eckit-2.0.7/0000775000175000017500000000000015161702250013002 5ustar alastairalastaireckit-2.0.7/VERSION0000664000175000017500000000000615161702250014046 0ustar alastairalastair2.0.7 eckit-2.0.7/NOTICE0000664000175000017500000000067415161702250013715 0ustar alastairalastaireckit ===== Copyright 1996- ECMWF This product is developed by the Development Section, European Centre for Medium-Range Weather Forecasts (ECMWF) - http://www.ecmwf.int Below is a list of software packages which are used inside eckit: - xxHash (www.xxhash.com) Copyright (c) 2012-2020 Yann Collet BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) See `src/eckit/contrib/xxhash` for more information eckit-2.0.7/regressions/0000775000175000017500000000000015161702250015345 5ustar alastairalastaireckit-2.0.7/regressions/ECKIT-166.cc0000664000175000017500000000104515161702250017025 0ustar alastairalastair/* BUG ECKIT-166 ============= */ #include #include "eckit/mpi/Comm.h" #include "eckit/runtime/Main.h" using namespace eckit; void run(int argc, char** argv) { Main::initialise(argc, argv); // This should initialise MPI automatically if (mpi::comm().rank() == 0) { std::cout << "mpi::comm().size() = " << mpi::comm().size() << std::endl; } } int main(int argc, char** argv) { std::cout << "begin of main" << std::endl; run(argc, argv); std::cout << "end of main" << std::endl; return 0; } eckit-2.0.7/regressions/ECKIT-221.cc0000664000175000017500000000211515161702250017014 0ustar alastairalastair/* FEATURE ECKIT-221 ================= Test broadcasting of file, read on one processor */ #include #include #include #include #include "eckit/io/DataHandle.h" #include "eckit/mpi/Comm.h" #include "eckit/runtime/Tool.h" #include "eckit/utils/Translator.h" using namespace eckit; class MyTool : public Tool { public: MyTool(int argc, char** argv) : Tool(argc, argv) {} virtual ~MyTool() { mpi::finaliseAllComms(); } virtual void run() { PathName filename(argv()[1]); SharedBuffer buffer = mpi::comm().broadcastFile(filename, 0); PathName out = filename + "." + Translator()(mpi::comm().rank()); std::unique_ptr dh(out.fileHandle()); AutoClose closer(*dh); dh->openForWrite(buffer.size()); dh->write(buffer.data(), buffer.size()); } }; int main(int argc, char** argv) { if (argc != 2) { std::cout << "Usage: " << argv[0] << " " << std::endl; return -1; } MyTool tool(argc, argv); return tool.start(); } eckit-2.0.7/regressions/ECKIT-175.sh.in0000775000175000017500000000106315161702250017462 0ustar alastairalastair#!/usr/bin/env bash set -eux cd @CMAKE_CURRENT_BINARY_DIR@ ### Case 1 rm -f ECKIT-175.out && ./ECKIT-175.x 1 out=$(cat ECKIT-175.out | grep "NON Flushed message in myrun") [[ -z ${out:-} ]] && echo "Failed" && exit 1 ### Case 2 rm -f ECKIT-175.out && ./ECKIT-175.x 2 out=$(cat ECKIT-175.out | grep "NON Flushed message in myrun") [[ -z ${out:-} ]] && echo "Failed" && exit 1 ### Case 3 rm -f ECKIT-175.out && ./ECKIT-175.x 3 out=$(cat ECKIT-175.out | grep "NON Flushed message in myrun") [[ -z ${out:-} ]] && echo "Failed" && exit 1 echo "OK" eckit-2.0.7/regressions/ECKIT-175.cc0000664000175000017500000000370315161702250017030 0ustar alastairalastair/* BUG ECKIT-175 ============= when Main::initialise(argc,argv) is *not* called or a Tool is *not* used, then everything is OK, and the FileTarget gets destructed as opposed to when Main::initialise(argc,argv) *is* called, or a Tool *is* used. Program expects an integer argument. Possible values: 0 : Test does *not* initialise a Main::instance() Everything works as expected. 1 : Test explicitely starts with Main::initialise(argc,argv) Buffers should be flushed but aren't 2 : Test uses a eckit::Tool to initialise a Main::instance() Buffers should be flushed but aren't Note: The destructor ~FileTarget() is modified to print a message when invoked, so don't merge this */ #include #include #include #include "eckit/log/Log.h" #include "eckit/runtime/Main.h" #include "eckit/runtime/Tool.h" using namespace eckit; void myrun() { Log::setFile("ECKIT-175.out"); Log::info() << "Flushed message in myrun()" << std::endl; Log::info() << "NON Flushed message in myrun()\n"; } class MyTool : public Tool { public: MyTool(int argc, char** argv) : Tool(argc, argv) {} virtual ~MyTool() {} virtual void run() { myrun(); } }; enum { NO_INIT = 1, INIT = 2, TOOL = 3 }; int main(int argc, char** argv) { if (argc != 2) { std::cout << "Usage: " << argv[0] << " " << std::endl; std::cout << " Case values: 1 2 3" << std::endl; return 1; } int test = std::atoi(argv[1]); switch (test) { case 1: { myrun(); break; } case 2: { Main::initialise(argc, argv); myrun(); break; } case 3: { MyTool tool(argc, argv); tool.start(); break; } default: exit(1); break; } return 0; } eckit-2.0.7/regressions/ECKIT-221.sh.in0000775000175000017500000000172415161702250017456 0ustar alastairalastair#!/usr/bin/env bash set -eux cd @CMAKE_CURRENT_BINARY_DIR@ exe="./ECKIT-221.x" fname="file1.txt" str="abc123" ### Case 1 - Serial function create_file { rm -f $fname* echo "$str" >> $fname echo "$str" >> $fname echo "$str" >> $fname } create_file $exe "$fname" cmp "$fname" "$fname.0" ### Case 2 - Parallel if [ "@eckit_HAVE_MPI@" == "1" ] && [ "@MPIEXEC@" != "MPIEXEC-NOTFOUND" ]; then create_file @MPIEXEC@ @MPI_ARGS@ @MPIEXEC_NUMPROC_FLAG@ 4 $exe "$fname" set +e filed_found=0 for i in $(awk 'BEGIN {for(i=0;i<40;i++) print i}'); do ls -l $fname.0 $fname.1 $fname.2 $fname.3; status=$? if [ "$status" -eq 0 ]; then files_found=1 break else sleep 5 fi echo "Files not yet found or not yet accessible. Retry ($i)" done set -e if [ "${files_found}" -eq 0 ]; then echo "Files not found or not accessible" exit 1 fi for i in 0 1 2 3; do cmp "$fname" "$fname.$i" done fi echo "OK" eckit-2.0.7/regressions/CMakeLists.txt0000664000175000017500000000050315161702250020103 0ustar alastairalastairlist( APPEND regressions ECKIT-175 ECKIT-221 ECKIT-166 ) foreach( r ${regressions} ) configure_file( ${r}.sh.in ${r}.sh @ONLY ) ecbuild_add_executable( NOINSTALL TARGET ${r}.x SOURCES ${r}.cc LIBS eckit_mpi eckit ) ecbuild_add_test( TYPE SCRIPT COMMAND ${r}.sh ) endforeach() eckit-2.0.7/regressions/ECKIT-166.sh.in0000775000175000017500000000035015161702250017460 0ustar alastairalastair#!/usr/bin/env bash set -eux cd @CMAKE_CURRENT_BINARY_DIR@ exe="./ECKIT-166.x" if [ "@eckit_HAVE_MPI@" == "1" ] && [ "@MPIEXEC@" != "MPIEXEC-NOTFOUND" ]; then @MPIEXEC@ @MPI_ARGS@ @MPIEXEC_NUMPROC_FLAG@ 1 $exe fi echo "OK" eckit-2.0.7/tests/0000775000175000017500000000000015161702250014144 5ustar alastairalastaireckit-2.0.7/tests/serialisation/0000775000175000017500000000000015161702250017012 5ustar alastairalastaireckit-2.0.7/tests/serialisation/test_file_stream.cc0000664000175000017500000001034215161702250022652 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/io/AutoCloser.h" #include "eckit/serialisation/FileStream.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- namespace { const char i_char('a'); const unsigned char i_uchar('b'); const bool i_bool(true); const int i_int(-20000000); const unsigned int i_uint(30000000); const short i_short(-4000); const unsigned short i_ushort(5000); const long i_long(-60000000); const unsigned long i_ulong(70000000); const long long i_longlong(-80000000); const unsigned long long i_ulonglong(90000000); // const float i_float(300000); const double i_double(100000000); const string i_string("abcdefghijklmnopqrstuvwx"); const char* i_charp("cdefghijklmnopqrstuvwxyz"); char v_char; unsigned char v_uchar; bool v_bool; int v_int; unsigned int v_uint; short v_short; unsigned short v_ushort; long v_long; unsigned long v_ulong; long long v_longlong; unsigned long long v_ulonglong; // float v_float; double v_double; string v_string; string v_charp; } // anonymous namespace struct F { ~F() { if (filename.exists()) { filename.unlink(); } } static PathName filename; }; PathName F::filename = PathName::unique("data"); //---------------------------------------------------------------------------------------------------------------------- CASE("write_data") { FileStream sout(F::filename, "w"); auto c = closer(sout); sout << i_char << i_uchar << i_bool << i_int << i_uint << i_short << i_ushort << i_long << i_ulong << i_longlong << i_ulonglong // << i_float << i_double << i_string << i_charp; } CASE("read_data") { FileStream sin(F::filename, "r"); auto c = closer(sin); sin >> v_char >> v_uchar >> v_bool >> v_int >> v_uint >> v_short >> v_ushort >> v_long >> v_ulong >> v_longlong >> v_ulonglong // >> v_float >> v_double >> v_string >> v_charp; } CASE("check_data") { EXPECT(v_char == i_char); EXPECT(v_uchar == i_uchar); EXPECT(v_bool == i_bool); EXPECT(v_int == i_int); EXPECT(v_uint == i_uint); EXPECT(v_short == i_short); EXPECT(v_ushort == i_ushort); EXPECT(v_long == i_long); EXPECT(v_ulong == i_ulong); EXPECT(v_longlong == i_longlong); EXPECT(v_ulonglong == i_ulonglong); // EXPECT( v_float == i_float ); EXPECT(v_double == i_double); EXPECT(v_string == i_string); EXPECT(v_charp == i_charp); } CASE("stream_object") { Log::info() << "Stream an object" << std::endl; const std::string k("key"); const std::string v("value"); { FileStream sout(F::filename, "w"); auto c = closer(sout); sout.startObject(); sout << k; sout << v; sout.endObject(); } { FileStream sin(F::filename, "r"); auto c = closer(sin); EXPECT(sin.next()); std::string s; sin >> s; EXPECT(s == k); sin >> s; EXPECT(s == v); EXPECT(sin.endObjectFound()); EXPECT(!sin.next()); } } CASE("stream_string") { Log::info() << "Stream a string" << std::endl; { FileStream sout(F::filename, "w"); auto c = closer(sout); sout << i_string; } { FileStream sin(F::filename, "r"); auto c = closer(sin); std::string s; EXPECT(sin.next(s)); EXPECT(s == i_string); EXPECT(!sin.next()); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/serialisation/CMakeLists.txt0000664000175000017500000000044315161702250021553 0ustar alastairalastairecbuild_add_test( TARGET eckit_test_serialisation_file_stream SOURCES test_file_stream.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_serialisation_streamable SOURCES test_streamable.cc LIBS eckit ) eckit-2.0.7/tests/serialisation/test_streamable.cc0000664000175000017500000002474515161702250022513 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. In applying * this licence, ECMWF does not waive the privileges and immunities granted to it by virtue * of its status as an intergovernmental organisation nor does it submit to any jurisdiction. */ /// @file test_streamable.cc /// @date Jan 2015 /// @author Florian Rathgeber #include #include #include #include "eckit/filesystem/PathName.h" #include "eckit/io/AutoCloser.h" #include "eckit/serialisation/FileStream.h" #include "eckit/serialisation/Streamable.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- template class TestItem : public Streamable { public: TestItem(const T& s) : payload_(s) {} TestItem(Stream& s) : Streamable(s), payload_() { s >> payload_; } // From Streamble virtual void encode(eckit::Stream& s) const { Streamable::encode(s); s << payload_; } virtual const eckit::ReanimatorBase& reanimator() const; // Class members static const eckit::ClassSpec& classSpec() { return classSpec_; } T payload_; protected: virtual void print(std::ostream& s) const { s << "TestItem " << payload_; } private: // -- Class members static eckit::ClassSpec classSpec_; // -- Friends friend std::ostream& operator<<(std::ostream& s, const TestItem& p) { p.print(s); return s; } }; template const ReanimatorBase& TestItem::reanimator() const { static Reanimator > reanimator; return reanimator; } using uchar = unsigned char; using llong = long long; #ifndef ulong using ulong = unsigned long; #endif using ullong = unsigned long long; /// Partially specialise ClassSpecs since they must have unique names template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemChar", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemUChar", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemBool", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemInt", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemUInt", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemShort", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemUShort", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemLong", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemULong", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemLLong", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemULLong", }; // NOTE: float is not implemented! // template <> // ClassSpec TestItem::classSpec_ = {&Streamable::classSpec(),"TestItemFloat",}; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemDouble", }; template <> ClassSpec TestItem::classSpec_ = { &Streamable::classSpec(), "TestItemString", }; //---------------------------------------------------------------------------------------------------------------------- #define test_decode(TYPE, INITIAL, SUFFIX) \ CASE("test_decode_" #TYPE "_" #SUFFIX) { \ Log::info() << "Manually (de)serialise Streamable with " #TYPE " member" << std::endl; \ PathName filename = PathName::unique("data"); \ std::string filepath = filename.asString(); \ TestItem t(INITIAL); \ { \ FileStream sout(filepath.c_str(), "w"); \ auto c = closer(sout); \ t.encode(sout); \ } \ { \ FileStream sin(filepath.c_str(), "r"); \ auto c = closer(sin); \ TestItem t2(sin); \ Log::info() << "original: " << t.payload_ << std::endl; \ Log::info() << "streamed: " << t2.payload_ << std::endl; \ EXPECT(t.payload_ == t2.payload_); \ } \ if (filename.exists()) \ filename.unlink(); \ } #define test_reanimate(TYPE, INITIAL, SUFFIX) \ CASE("test_reanimate_" #TYPE "_" #SUFFIX) { \ Log::info() << "(de)serialise Streamable with " #TYPE " member via Reanimator" << std::endl; \ PathName filename = PathName::unique("data"); \ std::string filepath = filename.asString(); \ TestItem t(INITIAL); \ { \ FileStream sout(filepath.c_str(), "w"); \ auto c = closer(sout); \ sout << t; \ } \ { \ FileStream sin(filepath.c_str(), "r"); \ auto c = closer(sin); \ TestItem* t2 = eckit::Reanimator >::reanimate(sin); \ Log::info() << "orginal: " << t.payload_ << std::endl; \ Log::info() << "streamed: " << t2->payload_ << std::endl; \ EXPECT(t.payload_ == t2->payload_); \ delete t2; \ } \ if (filename.exists()) \ filename.unlink(); \ } test_decode(char, 'A', max); test_decode(char, 'z', min); test_reanimate(char, 'A', max); test_reanimate(char, 'z', min); test_decode(uchar, 'A', max); test_reanimate(uchar, 'A', max); test_decode(bool, true, true); test_reanimate(bool, true, true); test_decode(bool, false, false); test_reanimate(bool, false, false); test_decode(int, numeric_limits::max(), max); test_decode(int, numeric_limits::min(), min); test_reanimate(int, numeric_limits::max(), max); test_reanimate(int, numeric_limits::min(), min); test_decode(uint, numeric_limits::max(), max); test_reanimate(uint, numeric_limits::max(), max); test_decode(short, numeric_limits::max(), max); test_decode(short, numeric_limits::min(), min); test_reanimate(short, numeric_limits::max(), max); test_reanimate(short, numeric_limits::min(), min); test_decode(ushort, numeric_limits::max(), max); test_reanimate(ushort, numeric_limits::max(), max); // NOTE: long in eckit is always 32 bit! test_decode(long, numeric_limits::max(), max); test_decode(long, numeric_limits::min(), min); test_reanimate(long, numeric_limits::max(), max); test_reanimate(long, numeric_limits::min(), min); test_decode(ulong, numeric_limits::max(), max); test_reanimate(ulong, numeric_limits::max(), max); test_decode(llong, numeric_limits::max(), max); test_decode(llong, numeric_limits::min(), min); test_reanimate(llong, numeric_limits::max(), max); test_reanimate(llong, numeric_limits::min(), min); test_decode(ullong, numeric_limits::max(), max); test_reanimate(ullong, numeric_limits::max(), max); // NOTE: float is not implemented! // test_decode(float, numeric_limits::max(), max); // test_decode(float, numeric_limits::min(), min); // test_reanimate(float, numeric_limits::max(), max); // test_reanimate(float, numeric_limits::min(), min); test_decode(double, numeric_limits::max(), max); test_decode(double, numeric_limits::min(), min); test_reanimate(double, numeric_limits::max(), max); test_reanimate(double, numeric_limits::min(), min); test_decode(string, "Hello, World!", Hello); test_reanimate(string, "Hello, World!", Hello); test_decode(string, "", empty); test_reanimate(string, "", empty); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/testing/0000775000175000017500000000000015161702250015621 5ustar alastairalastaireckit-2.0.7/tests/testing/test_testing.cc0000664000175000017500000002315015161702250020645 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/types/FloatCompare.h" #include "eckit/value/Value.h" #define ECKIT_TESTING_SELF_REGISTER_CASES 0 #include "eckit/testing/Test.h" // Disable warnings for old-style casts in these tests. They are intentional #ifdef __clang__ #pragma clang diagnostic ignored "-Wold-style-cast" #endif using namespace eckit::testing; namespace eckit_test { // Not eckit namespace on purpose to test downstream usage of macros using Tests = std::vector; //---------------------------------------------------------------------------------------------------------------------- // These unit tests test the eckit testing framework itself. //---------------------------------------------------------------------------------------------------------------------- // We need some global counters to keep count of some of the CASE functionality std::vector global_counters(5, 0); void ThrowStd() { throw std::exception(); } Tests tests = {{CASE("CASE with no EXPECT passes"){ }}, {CASE("EXPECT macros are defined correctly"){EXPECT(true); EXPECT_NOT(false); EXPECT_EQUAL(1, 1); EXPECT_NOT_EQUAL(1, 2); EXPECT_NO_THROW({ bool b = true; }); EXPECT_THROWS(throw std::exception()); EXPECT_THROWS_AS(throw std::exception(), std::exception); EXPECT_MSG(1 == 1, [=]() { std::cerr << eckit::Colour::red << "1 != 1" << eckit::Colour::reset << std::endl; };); } // namespace eckit_test } // namespace eckit , {CASE("EXPECT does not cause the test to fail on success"){ Test pass = {CASE("P"){EXPECT(true); } } ; std::vector f; bool ret = pass.run(TestVerbosity::Silent, f); if (!ret || f.size() != 0) { throw eckit::testing::TestException("Unexpected error encountered in EXPECT", Here()); } } } , {CASE("EXPECT causes an error to be reported on failure"){ Test fail = {CASE("F"){EXPECT(false); } } ; std::vector f; bool ret = fail.run(TestVerbosity::Silent, f); if (ret || f.size() == 0) { throw eckit::testing::TestException("No error reported when EXPECT failed", Here()); } if (ret || f.size() > 1) { throw eckit::testing::TestException("Too many errors reported when EXPECT failed", Here()); } } } , {CASE("EXPECT succeeds for success (true) and failure (false)"){Tests pass = {{CASE("P"){EXPECT(true); } } } ; Tests fail = {{CASE("F"){EXPECT(false); } } } ; EXPECT(0 == run(pass, TestVerbosity::Silent)); EXPECT(1 == run(fail, TestVerbosity::Silent)); } } , {CASE("EXPECT succeeds for integer comparison"){EXPECT(7 == 7); EXPECT(7 != 8); EXPECT(7 >= 6); EXPECT(7 <= 8); EXPECT(7 > 6); EXPECT(7 < 8); EXPECT_NOT(7 == 8); EXPECT_NOT(7 != 7); EXPECT_NOT(7 <= 6); EXPECT_NOT(7 >= 8); EXPECT_NOT(7 < 6); EXPECT_NOT(7 > 8); } } , {CASE("Expect succeeds for integer vs. real comparison"){EXPECT(7.0 == 7); EXPECT(7.0 != 8); EXPECT(7 == 7.0); EXPECT(7 != 8.0); EXPECT_NOT(7.0 == 8); EXPECT_NOT(7 != 7.0); } } , {CASE("Expect succeeds for string comparison"){ std::string a("a"); std::string b("b"); EXPECT(a == a); EXPECT(a != b); EXPECT(b >= a); EXPECT(a <= b); EXPECT(b > a); EXPECT(a < b); EXPECT_NOT(a == b); EXPECT_NOT(a != a); EXPECT_NOT(b <= a); EXPECT_NOT(a >= b); EXPECT_NOT(b < a); EXPECT_NOT(a > b); } } , {CASE("Expect expression RHS can use * / % + -"){EXPECT(7 == 1 * 7); EXPECT(7 == 7 / 1); EXPECT(0 == 7 % 1); EXPECT(7 == 1 + 6); EXPECT(7 == 8 - 1); } } , {CASE("Expect expression LHS can use * / % + -"){EXPECT(1 * 7 == 7); EXPECT(7 / 1 == 7); EXPECT(7 % 1 == 0); EXPECT(1 + 6 == 7); EXPECT(8 - 1 == 7); } } , {CASE("run() returns the correct failure count"){Tests pass = {{CASE("P"){EXPECT(1 == 1); } } } ; Tests fail_1 = {{CASE("F1"){EXPECT(0 == 1); } } } ; Tests fail_3 = {{CASE("F1"){EXPECT(0 == 1); } } , {CASE("F2"){EXPECT(0 == 1); } } , { CASE("F3") { EXPECT(0 == 1); } } } ; EXPECT(0 == run(pass, TestVerbosity::Silent)); EXPECT(1 == run(fail_1, TestVerbosity::Silent)); EXPECT(3 == run(fail_3, TestVerbosity::Silent)); } } , {CASE("run() returns -1 if no CASE() is found"){Tests empty = {}; EXPECT(-1 == run(empty, TestVerbosity::Silent)); } } , {CASE("A fresh fixture is created for each section"){ int i = 7; SECTION("S1") { i = 42; } SECTION("S2") { EXPECT(i == 7); } } } , {CASE("Deprecated SETUP macro may be used."){ SETUP("Context"){int i = 7; SECTION("S1") { i = 42; } SECTION("S2") { EXPECT(i == 7); } } } } , {CASE("Fixture setup runs one more time than the number of sections."){ ++global_counters[0]; SECTION("S1") { ++global_counters[1]; } SECTION("S2") { ++global_counters[2]; } } } , {CASE("Sections run correctly inside a for loop"){ ++global_counters[3]; for (int j = 0; j < 10; j++) { std::stringstream ss; ss << "test-" << j; SECTION(ss.str()) { ++global_counters[4]; } } } } , {CASE("Expect runs multiple times in for-loop"){ int i = 0; for (int j = 0; j < 10; j++) { EXPECT(i++ == j); } EXPECT(i == 10); } } , {CASE("Collections compare correctly"){ SETUP("Create Collections"){std::vector a = {1, 2, 3}; std::vector b = {1, 2, 3}; SECTION("Compare Collections") { EXPECT(a == b); EXPECT(a <= b); EXPECT(a >= b); EXPECT_NOT(a != b); EXPECT_NOT(a > b); EXPECT_NOT(a < b); a[0] = 0; EXPECT_NOT(a == b); EXPECT_NOT(a >= b); EXPECT(a <= b); EXPECT(a < b); EXPECT(a != b); a[0] = 2; EXPECT_NOT(a == b); EXPECT_NOT(a <= b); EXPECT(a >= b); EXPECT(a > b); EXPECT(a != b); } } } } , {CASE("Test comparisons of c-style arrays"){ std::vector a = {1, 2, 3}; std::vector b = {1, 2, 3}; int arr[3] = {1, 2, 3}; std::vector s = {"1", "22", "333"}; std::string arr_s[3] = {"1", "22", "333"}; std::vector d1 = {1., 2., 3.}; std::vector d2 = {1., 2., 3.}; // Check basic comparisons and initializations EXPECT(make_view(a.data(), a.data() + 3) == make_view(b.data(), b.data() + 3)); EXPECT(make_view(a.data(), 3) == make_view(b.data(), 3)); EXPECT(make_view(a.data(), a.data() + 3) == make_view(arr, arr + 3)); EXPECT(make_view(a.data(), 3) == make_view(arr, 3)); EXPECT(make_view(a) == make_view(arr, arr + 3)); EXPECT(make_view(s) == make_view(arr_s, arr_s + 3)); EXPECT(make_view(s.data(), s.data() + 3) == make_view(arr_s, arr_s + 3)); EXPECT(make_view(a) == make_view(b.data(), b.data() + 3)); EXPECT(make_view(a) == make_view(b)); EXPECT(make_view(d1.data(), 3) == make_view(d2.data(), 3)); EXPECT(make_view(d1.data(), 3) == make_view(d2.data(), 3)); EXPECT(make_view(a.data(), a.data() + 3).size() == 3); EXPECT(make_view(a.data(), 3).size() == 3); EXPECT(make_view(a).size() == a.size()); EXPECT(make_view(s).size() == s.size()); // Check sub-array comparisons EXPECT(make_view(a.data(), a.data() + 2) == make_view(b.data(), b.data() + 2)); EXPECT(make_view(&a[1], &a[1] + 2) == make_view(&b[1], &b[1] + 2)); EXPECT(make_view(&a[1], &a[1] + 2) != make_view(b.data(), b.data() + 2)); // values not equal EXPECT(make_view(a.data(), a.data() + 3) != make_view(b.data(), b.data() + 2)); // not same size EXPECT(make_view(a.data(), 2) == make_view(b.data(), 2)); EXPECT(make_view(a.data(), 2) == make_view(b.data(), 2)); // Check != is the same as !(==) EXPECT(!(make_view(a.data(), a.data() + 3) != make_view(b.data(), b.data() + 3))); EXPECT(!(make_view(a.data(), a.data() + 3) == make_view(b.data(), b.data() + 2))); // not same size EXPECT(!(make_view(&a[1], &a[1] + 2) == make_view(b.data(), b.data() + 2))); // values not equal // Check comparisons against std::vector EXPECT(a == make_view(b)); EXPECT(make_view(a) == b); // Check type recognition of make_view vs explicit constructor EXPECT(ArrayView(a) == make_view(a)); EXPECT(ArrayView(s) == make_view(s)); // Check approximately equals d2[2] = 2.99; EXPECT(is_approximately_equal(make_view(d1), make_view(d2), 0.02)); EXPECT(!is_approximately_equal(make_view(d1), make_view(d2), 0.0001)); // array vs array EXPECT(!is_approximately_equal(d1, d2, 0.0001)); // vector vs vector EXPECT(!is_approximately_equal(make_view(d1), d2, 0.0001)); // array vs vector EXPECT(!is_approximately_equal(d1, make_view(d2), 0.0001)); // vector vs array } } , } ; // end of tests //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit using eckit_test::global_counters; int main(int argc, char* argv[]) { int retval1 = 0; if (global_counters.size() != 5 || std::any_of(global_counters.begin(), global_counters.end(), [](int a) { return a != 0; })) { eckit::Log::info() << "Global counters incorrectly configured" << std::endl; retval1 = 1; } eckit::Main::initialise(argc, argv); int retval2 = eckit::testing::run_tests(eckit_test::tests, argc, argv); int retval3 = 0; if (global_counters[0] != 3 || global_counters[1] != 1 || global_counters[2] != 1 || global_counters[3] != 11 || global_counters[4] != 10) { eckit::Log::info() << "Global counters incorrect. Relevant tests FAILED" << std::endl; eckit::Log::info() << "Global counters: " << global_counters << std::endl; retval3 = 1; } return std::abs(retval1) + std::abs(retval2) + std::abs(retval3); } eckit-2.0.7/tests/testing/CMakeLists.txt0000664000175000017500000000017415161702250020363 0ustar alastairalastairecbuild_add_test( TARGET eckit_test_testing SOURCES test_testing.cc LIBS eckit ) eckit-2.0.7/tests/filesystem/0000775000175000017500000000000015161702250016330 5ustar alastairalastaireckit-2.0.7/tests/filesystem/test_pathname.cc0000664000175000017500000000266315161702250021502 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/filesystem/PathName.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Unknown path type fails") { EXPECT_THROWS_AS(PathName("unknown://a/path/component"), SeriousBug); } /* CASE("Creation of marsfs paths") { PathName m("marsfs://nodexxx/a/path/component"); EXPECT(m.node() == "nodexxx"); EXPECT(m.path() == "/a/path/component"); }*/ CASE("Creation of local paths") { PathName a("local:///an/absolute/path"); PathName b("/an/absolute/path"); PathName c("local://a/relative/path"); PathName d("a/relative/path"); EXPECT(a == b); EXPECT(c == d); EXPECT(a != c); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/test_aiohandle.cc0000664000175000017500000000534715161702250021633 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/AIOHandle.h" #include "eckit/io/AutoCloser.h" #include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/log/Log.h" #include "eckit/runtime/Tool.h" #include "eckit/testing/Test.h" #include "eckit/types/Types.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- // size is prime number 89 const char tbuf[] = "74e1feb8d0b1d328cbea63832c2dcfb2b4fa1adfeb8d0b1d328cb53d50e63a50fba73f0151028a695a238ff0"; class TestAIO { public: TestAIO() { std::string base = Resource("$TMPDIR", "/tmp"); path_ = PathName::unique(base + "/file") + ".dat"; reference_ = PathName::unique(base + "/reference") + ".dat"; std::unique_ptr fh(reference_.fileHandle()); auto close = closer(*fh); writeTo(*fh); } size_t writeTo(DataHandle& dh) { long sz = sizeof(tbuf); dh.openForWrite(0); size_t nblocks = 30 * 1024 * 1024 / sz; size_t total = 0; for (size_t i = 0; i < nblocks; ++i) { total += dh.write(tbuf, sz); } dh.close(); return total; } bool verify() { std::unique_ptr fh(path_.fileHandle()); std::unique_ptr rh(reference_.fileHandle()); return rh->compare(*fh); } ~TestAIO() { path_.unlink(); reference_.unlink(); } PathName path_; PathName reference_; }; CASE("Write to a new file") { TestAIO test; SECTION("Single write") { std::unique_ptr rh(test.reference_.fileHandle()); std::unique_ptr aioh(new AIOHandle(test.path_)); rh->saveInto(*aioh); EXPECT(test.verify()); } SECTION("Multiple writes write") { std::unique_ptr aioh(new AIOHandle(test.path_)); test.writeTo(*aioh); EXPECT(test.verify()); } } } // namespace eckit::test //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/test_filemode.cc0000664000175000017500000002066615161702250021474 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include #include #include "eckit/config/Resource.h" #include "eckit/filesystem/FileMode.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/filesystem/PathExpander.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" #include "eckit/os/AutoUmask.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Create a file controling its mode") { SECTION("---,---,---") { FileMode m(("---,---,---")); FileMode m2; EXPECT_NO_THROW(m2 = ("---,---,---")); EXPECT(m == m2); std::cout << m << " = " << std::oct << std::setw(4) << std::setfill('0') << m.mode() << std::dec << std::endl; EXPECT(m.str() == std::string(("---,---,---"))); } SECTION("rwx,rwx,rwx") { FileMode m("rwx,rwx,rwx"); FileMode m2; EXPECT_NO_THROW(m2 = "rwx,rwx,rwx"); EXPECT(m == m2); std::cout << m << " = " << std::oct << std::setw(4) << std::setfill('0') << m.mode() << std::dec << std::endl; EXPECT(m.str() == std::string("rwx,rwx,rwx")); } SECTION("rwx,---,---") { FileMode m("rwx,---,---"); FileMode m2; EXPECT_NO_THROW(m2 = "rwx,---,---"); EXPECT(m == m2); std::cout << m << " = " << std::oct << std::setw(4) << std::setfill('0') << m.mode() << std::dec << std::endl; EXPECT(m.str() == std::string("rwx,---,---")); } SECTION("---,rwx,---") { FileMode m("---,rwx,---"); FileMode m2; EXPECT_NO_THROW(m2 = "---,rwx,---"); EXPECT(m == m2); std::cout << m << " = " << std::oct << std::setw(4) << std::setfill('0') << m.mode() << std::dec << std::endl; EXPECT(m.str() == std::string("---,rwx,---")); } SECTION("---,---,rwx") { FileMode m("---,---,rwx"); FileMode m2; EXPECT_NO_THROW(m2 = "---,---,rwx"); EXPECT(m == m2); std::cout << m << " = " << std::oct << std::setw(4) << std::setfill('0') << m.mode() << std::dec << std::endl; EXPECT(m.str() == std::string("---,---,rwx")); } SECTION("rw-,r--,r--") { FileMode m("rw-,r--,r--"); FileMode m2; EXPECT_NO_THROW(m2 = "rw-,r--,r--"); EXPECT(m == m2); std::cout << m << " = " << std::oct << std::setw(4) << std::setfill('0') << m.mode() << std::dec << std::endl; EXPECT(m.str() == std::string("rw-,r--,r--")); } SECTION("rwx,r-x,r-x") { FileMode m("rwx,r-x,r-x"); FileMode m2; EXPECT_NO_THROW(m2 = "rwx,r-x,r-x"); EXPECT(m == m2); std::cout << m << " = " << std::oct << std::setw(4) << std::setfill('0') << m.mode() << std::dec << std::endl; EXPECT(m.str() == std::string("rwx,r-x,r-x")); } } CASE("Handle bad strings") { FileMode m; SECTION("Bad chars") { EXPECT_THROWS_AS(m = "Owx,rwx,rwx", BadValue); EXPECT_THROWS_AS(m = "rOx,rwx,rwx", BadValue); EXPECT_THROWS_AS(m = "rwO,rwx,rwx", BadValue); EXPECT_THROWS_AS(m = "rwx,Owx,rwx", BadValue); EXPECT_THROWS_AS(m = "rwx,rOx,rwx", BadValue); EXPECT_THROWS_AS(m = "rwx,rwO,rwx", BadValue); EXPECT_THROWS_AS(m = "rwx,rwx,Owx", BadValue); EXPECT_THROWS_AS(m = "rwx,rwx,rOx", BadValue); EXPECT_THROWS_AS(m = "rwx,rwx,rwO", BadValue); EXPECT_THROWS_AS(m = "w--,---,---", BadValue); EXPECT_THROWS_AS(m = "x--,---,---", BadValue); EXPECT_THROWS_AS(m = "-r-,---,---", BadValue); EXPECT_THROWS_AS(m = "-x-,---,---", BadValue); EXPECT_THROWS_AS(m = "--r,---,---", BadValue); EXPECT_THROWS_AS(m = "--w,---,---", BadValue); EXPECT_THROWS_AS(m = "---,w--,---", BadValue); EXPECT_THROWS_AS(m = "---,x--,---", BadValue); EXPECT_THROWS_AS(m = "---,-r-,---", BadValue); EXPECT_THROWS_AS(m = "---,-x-,---", BadValue); EXPECT_THROWS_AS(m = "---,--r,---", BadValue); EXPECT_THROWS_AS(m = "---,--w,---", BadValue); EXPECT_THROWS_AS(m = "---,---,w--", BadValue); EXPECT_THROWS_AS(m = "---,---,x--", BadValue); EXPECT_THROWS_AS(m = "---,---,-r-", BadValue); EXPECT_THROWS_AS(m = "---,---,-x-", BadValue); EXPECT_THROWS_AS(m = "---,---,--r", BadValue); EXPECT_THROWS_AS(m = "---,---,--w", BadValue); EXPECT_THROWS_AS(m = "0228", BadValue); } } CASE("Handle bad constructor") { EXPECT_THROWS_AS(FileMode m(10000), BadValue); } CASE("Create a file and set its permissions") { AutoUmask mask(0); PathName p("foo.txt"); std::unique_ptr dh(p.fileHandle()); dh->openForAppend(0); FileMode m("rw-,r--,r--"); p.chmod(m); dh->close(); EXPECT(FileMode::fromPath(p) == m); p.unlink(); } CASE("Octal mode") { FileMode mode1("0666"); FileMode mode2(0666); FileMode mode3("rw-,rw-,rw-"); FileMode mode4 = std::string("rw-,rw-,rw-"); std::cout << "mode1 = " << mode1 << " oct=" << std::oct << std::setw(4) << std::setfill('0') << mode1.mode() << std::dec << std::endl; std::cout << "mode2 = " << mode2 << " oct=" << std::oct << std::setw(4) << std::setfill('0') << mode2.mode() << std::dec << std::endl; std::cout << "mode3 = " << mode3 << " oct=" << std::oct << std::setw(4) << std::setfill('0') << mode3.mode() << std::dec << std::endl; std::cout << "mode4 = " << mode4 << " oct=" << std::oct << std::setw(4) << std::setfill('0') << mode4.mode() << std::dec << std::endl; EXPECT(mode1 == mode2); EXPECT(mode1 == mode3); EXPECT(mode1 == mode4); } CASE("Use as umask") { FileMode filemode(0644); EXPECT(filemode.mask() == 022); AutoUmask mask(filemode.mask()); SECTION("File creation") { PathName p("bar.txt"); std::unique_ptr dh(p.fileHandle()); dh->openForAppend(0); dh->close(); FileMode result = FileMode::fromPath(p); std::cout << "result = " << result << " oct=" << std::oct << std::setw(4) << std::setfill('0') << result.mode() << std::dec << std::endl; EXPECT(result == filemode); p.unlink(); } SECTION("Directory creation") { PathName p("mydir"); p.mkdir(); FileMode result = FileMode::fromPath(p); std::cout << "result = " << result << " oct=" << std::oct << std::setw(4) << std::setfill('0') << result.mode() << std::dec << std::endl; EXPECT(result == FileMode(0755)); // directories get added --x,--x,--x (0111) by default p.rmdir(); } } CASE("From resource use as mask") { eckit::FileMode filemode(eckit::Resource("testFileMode", std::string("0644"))); EXPECT(filemode.mask() == 022); AutoUmask mask(filemode.mask()); SECTION("File creation") { PathName p("bar.txt"); std::unique_ptr dh(p.fileHandle()); dh->openForAppend(0); dh->close(); FileMode result = FileMode::fromPath(p); std::cout << "result = " << result << " oct=" << std::oct << std::setw(4) << std::setfill('0') << result.mode() << std::dec << std::endl; EXPECT(result == filemode); p.unlink(); } SECTION("Directory creation") { PathName p("mydir"); p.mkdir(); FileMode result = FileMode::fromPath(p); std::cout << "result = " << result << " oct=" << std::oct << std::setw(4) << std::setfill('0') << result.mode() << std::dec << std::endl; EXPECT(result == FileMode(0755)); // directories get added --x,--x,--x (0111) by default p.rmdir(); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/test_asynchandle.cc0000664000175000017500000000510215161702250022165 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/AsyncHandle.h" #include "eckit/io/Buffer.h" #include "eckit/io/FileHandle.h" #include "eckit/log/Log.h" #include "eckit/runtime/Tool.h" #include "eckit/testing/Test.h" #include "eckit/types/Types.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- class TestAsyncHandle { public: void setup(); void teardown(); void test_write(); void test_append(); PathName path_; }; void TestAsyncHandle::test_write() { std::unique_ptr asynch(new AsyncHandle(path_.fileHandle())); asynch->openForWrite(0); const char buf[] = "74e1feb8d0b1d328cbea63832c2dcfb2b4fa1adf"; asynch->write(buf, sizeof(buf)); asynch->close(); std::unique_ptr fh(path_.fileHandle()); fh->openForRead(); Buffer buf2(1024); fh->read(buf2, buf2.size()); fh->close(); EXPECT(buf == std::string(buf2)); } void TestAsyncHandle::test_append() { std::unique_ptr asynch(new AsyncHandle(path_.fileHandle())); asynch->openForAppend(0); const char buf[] = "53d50e63a50fba73f0151028a695a238ff06491c"; asynch->write(buf, sizeof(buf)); asynch->close(); std::unique_ptr fh(path_.fileHandle()); fh->openForRead(); fh->seek(sizeof(buf)); Buffer buf2(1024); fh->read(buf2, buf2.size()); fh->close(); EXPECT(buf == std::string(buf2)); } void TestAsyncHandle::setup() { std::string base = Resource("$TMPDIR", "/tmp"); path_ = PathName::unique(base + "/lolo"); path_ += ".dat"; } void TestAsyncHandle::teardown() { path_.unlink(); } CASE("test_asynchandle") { TestAsyncHandle test; test.setup(); test.test_write(); test.test_append(); test.teardown(); } } // namespace eckit::test //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/test_pathexpander.cc0000664000175000017500000000545215161702250022367 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/filesystem/PathExpander.h" #include "eckit/filesystem/PathName.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Expand a CWD") { std::string s = "{CWD}/tmp/foo"; char* e = ::getenv("CURRENT_TEST_DIR"); EXPECT(e != nullptr); std::string r = std::string(e) + std::string("/tmp/foo"); LocalPathName px = PathExpander::expand(s); EXPECT(px.realName() == LocalPathName(r).realName()); } CASE("Expand using missing handler") { std::string s = "{FOO}/tmp/foo"; EXPECT_THROWS_AS(PathExpander::expand(s), eckit::UserError); } CASE("Expand an environment variable") { std::string s = "{ENVVAR?FOO}/tmp/bar"; SYSCALL(::setenv("FOO", "/foobar", 1)); std::string ps = PathExpander::expand(s); std::string pr = "/foobar/tmp/bar"; EXPECT(ps == pr); // paths dont exist, compare strings } static std::string write_file() { // write file contents char* e = ::getenv("CURRENT_TEST_DIR"); ASSERT(e != nullptr); PathName foo(e); foo /= "foo"; std::ofstream of(foo.asString().c_str(), std::ofstream::trunc); of << "/hoofa/lomp" << std::endl; of.close(); return foo.asString(); } CASE("Expand with contents of a file") { std::string foo = write_file(); std::string s = "/baz/{FILE?" + foo + "}/tmp/bar"; LocalPathName px = PathExpander::expand(s); std::string r = std::string("/baz/hoofa/lomp/tmp/bar"); EXPECT(px == LocalPathName(r)); // paths dont exist, compare strings } CASE("Expand multiple times, multiple paths") { std::string foo = write_file(); SYSCALL(::setenv("TDIR", "testdir", 1)); std::string s = "/baz/{ENVVAR?TDIR}/tmp/bar:{FILE?" + foo + "}/xxx"; LocalPathName px = PathExpander::expand(s); std::string r = std::string("/baz/testdir/tmp/bar:/hoofa/lomp/xxx"); EXPECT(px == LocalPathName(r)); // paths dont exist, compare strings } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/test_restarthandle.cc0000664000175000017500000001315015161702250022536 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/MultiHandle.h" #include "eckit/io/PartFileHandle.h" #include "eckit/io/Buffer.h" #include "eckit/io/FileHandle.h" #include "eckit/io/HandleHolder.h" #include "eckit/log/Log.h" #include "eckit/runtime/Tool.h" #include "eckit/testing/Test.h" #include "eckit/types/Types.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- class Restart : public DataHandle, public HandleHolder { Length total_; Length nextStop_; public: static size_t increment() { return 77773; } // a prime larger than 4 KiB Restart(DataHandle* h) : HandleHolder(h), total_(0) { nextStop_ = increment(); } virtual Length openForRead() { NOTIMP; } virtual void openForWrite(const Length& len) { return handle().openForWrite(len); } virtual void openForAppend(const Length& len) { NOTIMP; } virtual long read(void* buffer, long len) { NOTIMP; } virtual long write(const void* buffer, long len) { if (total_ > nextStop_) { nextStop_ += len + increment(); // 67108879 is first prime after 64*1024*1024 -- the default buffer size in saveInto() // this way we test that we roll back to data position before the whole buffer size Offset backTo = std::max(total_ - Length(67108879), (long long int)0); std::cout << "backTo " << backTo << " nextStop " << nextStop_ << std::endl; throw RestartTransfer(backTo); } total_ += len; std::cout << "write @ " << total_ << std::endl; return handle().write(buffer, len); } virtual void close() { handle().close(); } virtual void flush() { NOTIMP; } virtual void rewind() { NOTIMP; } virtual void print(std::ostream& os) const { os << "Restart"; } virtual void skip(const Length&) { NOTIMP; } virtual Offset seek(const Offset&) { NOTIMP; } virtual Length estimate() { NOTIMP; } virtual Offset position() { NOTIMP; } virtual DataHandle* clone() const { NOTIMP; } virtual void restartReadFrom(const Offset& offset) { handle().restartReadFrom(offset); } virtual void restartWriteFrom(const Offset& offset) { handle().restartWriteFrom(offset); } }; //---------------------------------------------------------------------------------------------------------------------- class Tester { public: void setup(); void teardown(); void test_write(); PathName path1_; PathName path2_; PathName path3_; }; void Tester::test_write() { const char buf1[] = "abcdefghijklmnopqrstuvwxyz"; const char buf2[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; const size_t N = 1024 * 1024 * 10; // create first file { Buffer b1(N * 26); char* p = b1; for (size_t i = 0; i < N; i++) { memcpy(p, buf1, 26); p += 26; } FileHandle f1(path1_); f1.openForWrite(0); f1.write(b1, b1.size()); f1.close(); std::cout << path1_ << std::endl; } // create second file { Buffer b2(N * 26); char* p = b2; for (size_t i = 0; i < N; i++) { memcpy(p, buf2, 26); p += 26; } FileHandle f2(path2_); f2.openForWrite(0); f2.write(b2, b2.size()); f2.close(); std::cout << path2_ << std::endl; } std::cout << "-------------------------------------------------------" << std::endl; MultiHandle mh1; { for (int i = 0; i < 26; i++) { mh1 += new PartFileHandle(path1_, i * N, N); mh1 += new PartFileHandle(path2_, i * N, N); } std::cout << mh1 << " " << mh1.estimate() << std::endl; // mh1.compress(); // std::cout << mh1 << " " << mh1.estimate() << std::endl; std::cout << Here() << std::endl; Restart f3(path3_.fileHandle()); std::cout << f3 << std::endl; std::cout << Here() << std::endl; mh1.saveInto(f3); std::cout << Here() << std::endl; } DataHandle* fh = path3_.fileHandle(); EXPECT(fh->compare(mh1)); delete fh; } void Tester::setup() { std::string base = Resource("$TMPDIR", "/tmp"); path1_ = PathName::unique(base + "/path1"); path1_ += ".dat"; path2_ = PathName::unique(base + "/path2"); path2_ += ".dat"; path3_ = PathName::unique(base + "/path3"); path3_ += ".dat"; } void Tester::teardown() { if (path1_.exists()) { path1_.unlink(); } if (path2_.exists()) { path2_.unlink(); } if (path3_.exists()) { path3_.unlink(); } } CASE("test_restarthandle") { Tester test; test.setup(); try { test.test_write(); } catch (...) { test.teardown(); throw; } test.teardown(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/test_atomic_file_update.cc0000664000175000017500000000431115161702250023512 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" #include "eckit/log/JSON.h" #include "eckit/parser/JSONParser.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Atomically update the contents of a file") { std::ostringstream s; JSON json(s); json.startObject(); json << "restart" << true; json << "step" << int(42); json << "description" << "forecast run"; json.endObject(); Log::info() << s.str() << std::endl; PathName path = LocalPathName::cwd() + "/tmp/control.json"; PathName tmppath = PathName::unique(path); Log::info() << path << std::endl; Log::info() << tmppath << " : exists " << tmppath.exists() << std::endl; std::ofstream of(tmppath.localPath()); if (!of) { throw CantOpenFile(tmppath.localPath(), Here()); } of << s.str(); of.close(); PathName::rename(tmppath, path); Log::info() << path << " : exists " << path.exists() << std::endl; Value j = JSONParser::decodeFile(path); EXPECT(j.isOrderedMap()); EXPECT(j["restart"].isBool()); EXPECT(j["restart"].as() == true); EXPECT(j["step"].isNumber()); EXPECT(int(j["step"]) == 42); EXPECT(j["description"].isString()); EXPECT(j["description"].as() == std::string("forecast run")); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/test_uri.cc0000664000175000017500000004725215161702250020507 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/filesystem/URI.h" #include "eckit/io/Buffer.h" #include "eckit/serialisation/ResizableMemoryStream.h" #include "eckit/testing/Test.h" #include "eckit/types/Types.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Parsing uri (path)") { { URI uri("path"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "path"); EXPECT(uri.path() == "local://path"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix:path"); EXPECT(uri.asString() == "path"); } { URI uri("//:123"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "//:123"); EXPECT(uri.path() == "local:///:123"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix://:123"); EXPECT(uri.asString() == "//:123"); } { URI uri(":path"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "path"); EXPECT(uri.path() == "local://path"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix:path"); EXPECT(uri.asString() == "path"); } { URI uri(":path1:path2"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "path1:path2"); EXPECT(uri.path() == "local://path1:path2"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix:path1:path2"); EXPECT(uri.asString() == "path1:path2"); } { URI uri(":/folder1/folder2/file"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "/folder1/folder2/file"); EXPECT(uri.path() == "local:///folder1/folder2/file"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix:/folder1/folder2/file"); EXPECT(uri.asString() == "/folder1/folder2/file"); } { URI uri("foo:"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "foo:"); EXPECT(uri.path() == "local://foo:"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix:foo:"); EXPECT(uri.asString() == "foo:"); } { URI uri("!"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "!"); EXPECT(uri.path() == "local://!"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix:!"); EXPECT(uri.asString() == "!"); } { URI uri("file:!"); EXPECT(uri.scheme() == "file"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "!"); EXPECT(uri.path() == "local://!"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "file:!"); EXPECT(uri.asString() == "!"); } { // relative path URI uri("file:path"); EXPECT(uri.scheme() == "file"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "path"); EXPECT(uri.path() == "local://path"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "file:path"); EXPECT(uri.asString() == "path"); } { URI uri(":http://nodename/path"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "http://nodename/path"); EXPECT(uri.path() == "local://http://nodename/path"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix:http://nodename/path"); EXPECT(uri.asString() == "http://nodename/path"); } } CASE("Parsing uri (scheme)") { { URI uri("file:///path"); EXPECT(uri.scheme() == "file"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "///path"); EXPECT(uri.path() == "/path"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "file:///path"); EXPECT(uri.asString() == "///path"); } { URI uri("/this/is/a/path:with:colons"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "/this/is/a/path:with:colons"); EXPECT(uri.path() == "/this/is/a/path:with:colons"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix:/this/is/a/path:with:colons"); EXPECT(uri.asString() == "/this/is/a/path:with:colons"); } } CASE("Parsing uri (query & fragment)") { { URI uri("file://host:123/path?query#fragment"); EXPECT(uri.scheme() == "file"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "//host:123/path"); EXPECT(uri.path() == "/host:123/path"); EXPECT(uri.query().empty()); EXPECT(uri.fragment() == "fragment"); EXPECT(uri.asRawString() == "file://host:123/path#fragment"); EXPECT(uri.asString() == "//host:123/path"); } { URI uri("file:///path?length=123&foo=bar#fragment"); EXPECT(uri.scheme() == "file"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "///path"); EXPECT(uri.path() == "/path"); EXPECT(uri.query() == "foo=bar&length=123"); EXPECT(uri.query("foo") == "bar"); EXPECT(uri.query("length") == "123"); EXPECT(uri.query("non existing").empty()); EXPECT(uri.fragment() == "fragment"); EXPECT(uri.asRawString() == "file:///path?foo=bar&length=123#fragment"); EXPECT(uri.asString() == "///path"); } { URI uri("file:///path?length=123&foo=bar"); EXPECT(uri.scheme() == "file"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "///path"); EXPECT(uri.path() == "/path"); EXPECT(uri.query() == "foo=bar&length=123"); EXPECT(uri.query("foo") == "bar"); EXPECT(uri.query("length") == "123"); EXPECT(uri.query("non existing").empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "file:///path?foo=bar&length=123"); EXPECT(uri.asString() == "///path"); } { URI uri("file:///path?length=123&remapKey=expver%3D0001%2Cstream%3Doper"); EXPECT(uri.scheme() == "file"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "///path"); EXPECT(uri.path() == "/path"); EXPECT(uri.query() == "length=123&remapKey=expver%3D0001%2Cstream%3Doper"); EXPECT(uri.query("remapKey") == "expver=0001,stream=oper"); EXPECT(uri.query("length") == "123"); EXPECT(uri.query("non existing").empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "file:///path?length=123&remapKey=expver%3D0001%2Cstream%3Doper"); EXPECT(uri.asString() == "///path"); } } CASE("Parsing uri (authority)") { { URI uri("http://username:password@host:123/path"); EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "username:password@host:123"); EXPECT(uri.user() == "username:password"); EXPECT(uri.host() == "host"); EXPECT(uri.hostport() == "host:123"); EXPECT(uri.port() == 123); EXPECT(uri.name() == "/path"); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://username:password@host:123/path"); EXPECT(uri.asString() == "http://username:password@host:123/path"); } { URI uri("http//:123"); EXPECT(uri.scheme() == "unix"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.hostport().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "http//:123"); EXPECT(uri.path() == "local://http//:123"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "unix:http//:123"); EXPECT(uri.asString() == "http//:123"); } { URI uri("http://username:password@host:123path"); EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "username:password@host:123"); EXPECT(uri.user() == "username:password"); EXPECT(uri.host() == "host"); EXPECT(uri.hostport() == "host:123"); EXPECT(uri.port() == 123); EXPECT(uri.name() == "path"); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://username:password@host:123path"); EXPECT(uri.asString() == "http://username:password@host:123path"); } { URI uri("http://host:path"); EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "host:"); EXPECT(uri.user().empty()); EXPECT(uri.host() == "host"); EXPECT(uri.hostport() == "host:"); EXPECT(uri.port() == 0); EXPECT(uri.name() == "path"); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://host:path"); EXPECT(uri.asString() == "http://host:path"); } { URI uri("http://www.ecmwf.int"); EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "www.ecmwf.int"); EXPECT(uri.user().empty()); EXPECT(uri.host() == "www.ecmwf.int"); EXPECT(uri.hostport() == "www.ecmwf.int"); EXPECT(uri.port() == -1); EXPECT(uri.name().empty()); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://www.ecmwf.int"); EXPECT(uri.asString() == "http://www.ecmwf.int"); } { URI uri("http://www.ecmwf.int:80"); EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "www.ecmwf.int:80"); EXPECT(uri.user().empty()); EXPECT(uri.host() == "www.ecmwf.int"); EXPECT(uri.hostport() == "www.ecmwf.int:80"); EXPECT(uri.port() == 80); EXPECT(uri.name().empty()); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://www.ecmwf.int:80"); EXPECT(uri.asString() == "http://www.ecmwf.int:80"); } { URI uri("http://www.ecmwf.int/"); EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "www.ecmwf.int"); EXPECT(uri.user().empty()); EXPECT(uri.host() == "www.ecmwf.int"); EXPECT(uri.hostport() == "www.ecmwf.int"); EXPECT(uri.port() == -1); EXPECT(uri.name() == "/"); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://www.ecmwf.int/"); EXPECT(uri.asString() == "http://www.ecmwf.int/"); } { URI uri("http://host"); EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "host"); EXPECT(uri.user().empty()); EXPECT(uri.host() == "host"); EXPECT(uri.hostport() == "host"); EXPECT(uri.port() == -1); EXPECT(uri.name().empty()); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://host"); EXPECT(uri.asString() == "http://host"); } { URI uri("http://nodename/path"); EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "nodename"); EXPECT(uri.user().empty()); EXPECT(uri.host() == "nodename"); EXPECT(uri.hostport() == "nodename"); EXPECT(uri.port() == -1); EXPECT(uri.name() == "/path"); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://nodename/path"); EXPECT(uri.asString() == "http://nodename/path"); } { URI uri("https://localhost:123/path"); EXPECT(uri.scheme() == "https"); EXPECT(uri.authority() == "localhost:123"); EXPECT(uri.user().empty()); EXPECT(uri.host() == "localhost"); EXPECT(uri.hostport() == "localhost:123"); EXPECT(uri.port() == 123); EXPECT(uri.name() == "/path"); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "https://localhost:123/path"); EXPECT(uri.asString() == "https://localhost:123/path"); } /* { URI uri("marsfs://nodename/path"); EXPECT(uri.scheme() == "marsfs"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "//nodename/path"); EXPECT(uri.path() == "marsfs://nodename/path"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "marsfs://nodename/path"); EXPECT(uri.asString() == "marsfs://nodename/path"); } { URI uri("marsfs://localhost:123/path"); EXPECT(uri.scheme() == "marsfs"); EXPECT(uri.authority().empty()); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "//localhost:123/path"); EXPECT(uri.path() == "marsfs://localhost:123/path"); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "marsfs://localhost:123/path"); EXPECT(uri.asString() == "marsfs://localhost:123/path"); }*/ } CASE("Stream") { { URI uriOrig("http://username:password@host:123/path"); eckit::Buffer b(1000); // should be enough b.zero(); eckit::ResizableMemoryStream s(b); s << uriOrig; s.rewind(); URI uri(s); EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "username:password@host:123"); EXPECT(uri.user() == "username:password"); EXPECT(uri.host() == "host"); EXPECT(uri.port() == 123); EXPECT(uri.name() == "/path"); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://username:password@host:123/path"); EXPECT(uri.asString() == "http://username:password@host:123/path"); } { URI uriOrig("http://username:password@host:123/path"); eckit::Buffer b(1000); // should be enough b.zero(); eckit::ResizableMemoryStream s(b); s << uriOrig; s.rewind(); URI uri("/"); s >> uri; EXPECT(uri.scheme() == "http"); EXPECT(uri.authority() == "username:password@host:123"); EXPECT(uri.user() == "username:password"); EXPECT(uri.host() == "host"); EXPECT(uri.port() == 123); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query().empty()); EXPECT(uri.fragment().empty()); EXPECT(uri.asRawString() == "http://username:password@host:123/path"); EXPECT(uri.asString() == "http://username:password@host:123/path"); } { URI uriOrig("http:///path?length=123&foo=bar#fragment"); eckit::Buffer b(1000); // must be enough b.zero(); eckit::ResizableMemoryStream s(b); s << uriOrig; s.rewind(); URI uri(s); EXPECT(uri.scheme() == "http"); EXPECT(uri.user().empty()); EXPECT(uri.host().empty()); EXPECT(uri.port() == -1); EXPECT(uri.name() == "/path"); EXPECT_THROWS_AS(uri.path(), NotImplemented); EXPECT(uri.query() == "foo=bar&length=123"); EXPECT(uri.query("foo") == "bar"); EXPECT(uri.query("length") == "123"); EXPECT(uri.query("non existing").empty()); EXPECT(uri.fragment() == "fragment"); EXPECT(uri.asRawString() == "http:/path?foo=bar&length=123#fragment"); EXPECT(uri.asString() == "http:/path?foo=bar&length=123#fragment"); uri.endpoint(eckit::net::Endpoint("host", 123)); EXPECT(uri.host() == "host"); EXPECT(uri.port() == 123); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/test_localpathname.cc0000664000175000017500000003105015161702250022505 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/types/Types.h" #include "eckit/filesystem/FileSystemSize.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/filesystem/PathName.h" #include "eckit/filesystem/TmpDir.h" #include "eckit/filesystem/TmpFile.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/utils/Hash.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Building of paths") { EXPECT(LocalPathName().path() == "/"); LocalPathName p1("/fred/bill"); EXPECT(p1 == "/fred/bill"); LocalPathName p2(p1); EXPECT(p1 == p2); } CASE("Assignement of paths") { LocalPathName p; LocalPathName pd; p = pd; EXPECT(p.path() == "/"); LocalPathName p1("/fred/bill"); LocalPathName p2; p2 = p1; EXPECT(p2.path() == "/fred/bill"); LocalPathName p3; p3 = "/fred"; EXPECT(p3.path() == "/fred"); LocalPathName p4; p4 = std::string("/martin"); EXPECT(p4.path() == "/martin"); } CASE("Contactenation of paths") { LocalPathName p; p += "fred"; EXPECT(p.path() == "/fred"); p += "/joe/90"; EXPECT(p.path() == "/fred/joe/90"); // preserve trailing '/' p += '/'; EXPECT(p == "/fred/joe/90/"); } CASE("Extract dirname") { LocalPathName p("/fred/bill"); EXPECT(p.dirName() == "/fred"); EXPECT(p.fullName() == "/fred/bill"); // when no leading '/' on pathname, we append cwd to path, for fullName() LocalPathName p2("fred"); LocalPathName expected = LocalPathName::cwd() + LocalPathName("/fred"); EXPECT(p2.fullName() == expected); } CASE("Check a path exists and is a directory") { LocalPathName cwd = LocalPathName::cwd(); EXPECT(cwd.exists()); EXPECT(cwd.isDir()); LocalPathName acwd = LocalPathName::cwd(); EXPECT(cwd.sameAs(acwd)); Log::info() << "cwd " << cwd << std::endl; Log::info() << "cwd.mountPoint() " << cwd.mountPoint() << std::endl; Log::info() << "cwd.realName() " << cwd.realName() << std::endl; Log::info() << "cwd.node() " << cwd.node() << std::endl; Log::info() << "cwd.path() " << cwd.path() << std::endl; Log::info() << "size " << cwd.size() << std::endl; Log::info() << "lastAccess " << cwd.lastAccess() << std::endl; Log::info() << "lastModified " << cwd.lastModified() << std::endl; Log::info() << "created " << cwd.created() << std::endl; } CASE("Construct relative paths") { LocalPathName foobar("/foo/bar/1/2"); LocalPathName foozing("/foo/zing/3"); LocalPathName baz("/baz/koko/4"); LocalPathName root("/"); // shorter relative to longer LocalPathName r1 = foozing.relativePath(foobar); EXPECT(r1 == LocalPathName("../../../zing/3")); // longer relative to shorter LocalPathName r2 = foobar.relativePath(foozing); EXPECT(r2 == LocalPathName("../../bar/1/2")); // same relative to same LocalPathName r3 = foobar.relativePath(foobar); EXPECT(r3 == LocalPathName(".")); // relative to root EXPECT(foobar.relativePath("/") == LocalPathName("foo/bar/1/2")); // root relative to path EXPECT(root.relativePath(foobar) == LocalPathName("../../../..")); } CASE("We can touch files/directories to update the timestamp") { TmpDir d; PathName foo = d / "foo"; PathName bar = d / "bar"; PathName baz = d / "baz"; foo.mkdir(); EXPECT(foo.exists()); EXPECT_NO_THROW(foo.touch()); // updates the timestamp EXPECT(!bar.exists()); EXPECT_NO_THROW(bar.touch()); // doesn't exist, so creates a file EXPECT(bar.exists()); // time resolution is 1 second, so wait slightly longer ::usleep(1000001); EXPECT(!baz.exists()); EXPECT_NO_THROW(baz.touch()); // doesn't exist, so creates a file EXPECT(baz.exists()); EXPECT(bar.lastAccess() < baz.lastAccess()); EXPECT(bar.lastModified() < baz.lastModified()); // update timestamp of older file ::usleep(1000001); EXPECT_NO_THROW(bar.touch()); // exists, so updates last accessed time EXPECT(bar.lastAccess() > baz.lastAccess()); EXPECT(bar.lastModified() > baz.lastModified()); } CASE("Find children files and dirs") { std::set ref_dirs; ref_dirs.insert("testdir/foo"); ref_dirs.insert("testdir/foo/1"); ref_dirs.insert("testdir/foo/2"); ref_dirs.insert("testdir/foo/2/1"); ref_dirs.insert("testdir/bar"); ref_dirs.insert("testdir/baz"); std::set ref_files; ref_files.insert("testdir/foo/file.1"); ref_files.insert("testdir/foo/file.2"); ref_files.insert("testdir/bar/file.3"); for (std::set::const_iterator j = ref_files.begin(); j != ref_files.end(); ++j) { j->touch(); } PathName t(LocalPathName::cwd()); t += "/testdir"; std::vector files; std::vector dirs; SECTION("Find in one level") { t.children(files, dirs); EXPECT(files.size() == 0); EXPECT(dirs.size() == 3); std::cout << files << std::endl; std::cout << dirs << std::endl; for (std::vector::const_iterator j = dirs.begin(); j != dirs.end(); ++j) { LocalPathName d(j->localPath()); LocalPathName r = d.relativePath(LocalPathName::cwd()); std::cout << "relative path " << r << std::endl; EXPECT(ref_dirs.find(r) != ref_dirs.end()); } } SECTION("Find recursively") { t.childrenRecursive(files, dirs); EXPECT(files.size() == 3); EXPECT(dirs.size() == 6); std::cout << files << std::endl; std::cout << dirs << std::endl; for (std::vector::const_iterator j = dirs.begin(); j != dirs.end(); ++j) { LocalPathName d(j->localPath()); LocalPathName r = d.relativePath(LocalPathName::cwd()); std::cout << "relative path " << r << std::endl; EXPECT(ref_dirs.find(r) != ref_dirs.end()); } for (std::vector::const_iterator j = files.begin(); j != files.end(); ++j) { LocalPathName d(j->localPath()); LocalPathName r = d.relativePath(LocalPathName::cwd()); std::cout << "relative path " << r << std::endl; EXPECT(ref_files.find(r) != ref_files.end()); } } } CASE("Extract basename") { LocalPathName p1; EXPECT(p1.baseName(false) == ""); EXPECT(p1.baseName(true) == ""); LocalPathName p2("fred"); EXPECT(p2.baseName(false) == "fred"); EXPECT(p2.baseName(true) == "fred"); LocalPathName p("/a/made/up/path/that/does/not/exist/file.ok"); EXPECT(p.baseName(false) == "file"); EXPECT(p.baseName(true) == "file.ok"); } CASE("Extract extension from Path") { EXPECT(LocalPathName().extension() == ""); EXPECT(LocalPathName("fred").extension() == ""); EXPECT(LocalPathName("/path/to/fred").extension() == ""); EXPECT(LocalPathName("/path/with.dot/to/fred").extension() == ""); EXPECT(LocalPathName("fred.").extension() == "."); EXPECT(LocalPathName("fred..").extension() == "."); EXPECT(LocalPathName("/path/to/fred.ext").extension() == ".ext"); EXPECT(LocalPathName("/path/with.dot/to/fred.ext").extension() == ".ext"); } CASE("Calculate file system size") { LocalPathName cwd = LocalPathName::cwd(); FileSystemSize fs; cwd.fileSystemSize(fs); Log::info() << "cwd " << cwd << std::endl; Log::info() << "fs.available " << fs.available << std::endl; Log::info() << "fs.total " << fs.total << std::endl; EXPECT(fs.available > 0); EXPECT(fs.total > 0); } CASE("Create a unique path") { LocalPathName unique = LocalPathName::unique(LocalPathName::cwd()); unique.mkdir(); EXPECT(unique.exists()); unique.rmdir(); EXPECT(!unique.exists()); } CASE("Check link") { LocalPathName unique = LocalPathName::unique(LocalPathName::cwd()); unique.mkdir(); LocalPathName linkpath = LocalPathName::cwd() + "link"; ::symlink(unique.c_str(), linkpath.c_str()); EXPECT(linkpath.exists()); EXPECT(linkpath.isLink()); unique.rmdir(); linkpath.unlink(); } CASE("Special characters not expanded in baseName") { LocalPathName p = "/base/path/~filename"; EXPECT(p.dirName() == "/base/path"); EXPECT(std::string(p.baseName()) == "~filename"); } std::string tidy(const std::string& p) { return LocalPathName(p).path(); } CASE("Tidy a path") { SECTION("Constants") { EXPECT(tidy("") == ""); EXPECT(tidy(".") == "."); EXPECT(tidy("..") == ".."); EXPECT(tidy("/") == "/"); EXPECT(tidy("a") == "a"); EXPECT(tidy("/a") == "/a"); EXPECT(tidy("aaa") == "aaa"); EXPECT(tidy("/aaa") == "/aaa"); EXPECT(tidy("a/b") == "a/b"); EXPECT(tidy("/a/b") == "/a/b"); EXPECT(tidy("/a/b/") == "/a/b/"); } SECTION("Basic rules") { EXPECT(tidy("a/") == "a/"); EXPECT(tidy("/a/.") == "/a"); EXPECT(tidy("/a/b/..") == "/a"); EXPECT(tidy("////") == "/"); EXPECT(tidy("./.") == "."); EXPECT(tidy("./..") == ".."); EXPECT(tidy("../..") == "../.."); EXPECT(tidy("../.") == ".."); } SECTION("Cleanup unnecessary tokens") { EXPECT(tidy("/a/./b/../../c/") == "/c/"); EXPECT(tidy("/a/./b/../../c") == "/c"); EXPECT(tidy("/../../../../../a") == "/a"); EXPECT(tidy("/a/./b/./c/./d") == "/a/b/c/d"); EXPECT(tidy("./../foo.bar") == "../foo.bar"); // ECKIT-421 EXPECT(tidy(".//../foo.bar") == "../foo.bar"); EXPECT(tidy("..//foo.bar") == "../foo.bar"); EXPECT(tidy("././././././..//foo.bar") == "../foo.bar"); EXPECT(tidy("/a//b//c//////d") == "/a/b/c/d"); } SECTION("Merging of tokens") { EXPECT(tidy("/a//b/") == "/a/b/"); EXPECT(tidy("/a/..//foo.bar") == "/foo.bar"); EXPECT(tidy("/a/../b/foo.bar") == "/b/foo.bar"); EXPECT(tidy("/a/b/../../c/foo.bar") == "/c/foo.bar"); EXPECT(tidy("/a/../b/../c/../d") == "/d"); EXPECT(tidy("../a/../b/../c/../d") == "../d"); EXPECT(tidy("a/../b/../c/../d") == "d"); EXPECT(tidy("/a/./b/./c/./d") == "/a/b/c/d"); } SECTION("Absolute paths") { EXPECT(tidy("/a/..") == "/"); EXPECT(tidy("/.") == "/"); } SECTION("Relative paths") { EXPECT(tidy("../a") == "../a"); EXPECT(tidy("a/b/c/..") == "a/b"); EXPECT(tidy("a/b/c/../../..") == ""); EXPECT(tidy("a/../b/../c") == "c"); EXPECT(tidy("a/../b/../c/..") == ""); EXPECT(tidy("a/../b/../c/../../..") == "../.."); EXPECT(tidy("../a/b/../../c") == "../c"); EXPECT(tidy("../a/../b/../../c") == "../../c"); } SECTION("Odd ones") { EXPECT(tidy("/../") == "/"); EXPECT(tidy("/./../foo.bar") == "/foo.bar"); EXPECT(tidy("/a/b/../../../c/foo.bar") == "/c/foo.bar"); EXPECT(tidy("../a/b/../../../c/foo.bar") == "../../c/foo.bar"); EXPECT(tidy("/a/../.././../../.") == "/"); } } CASE("Test PathName hashing") { // Create and hash some random data. Note that the buffer is bigger than and a non-integer // multiple of the buffer size used in hash generation #if eckit_HAVE_XXHASH const char* hash_method = "xxh64"; #else const char* hash_method = "MD5"; #endif Buffer tmp(131 * 1024 * 1024); for (int i = 0; i < tmp.size(); ++i) { tmp[i] = static_cast(random() % 255); } std::unique_ptr h(HashFactory::instance().build(hash_method)); h->add(tmp, tmp.size()); // Write the data to a file TmpFile file; { std::unique_ptr dh(file.fileHandle()); dh->openForWrite(0); AutoClose closer(*dh); dh->write(tmp, tmp.size()); } // Check that the hash matches EXPECT(h->digest() == file.hash(hash_method)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/test_filespace_strategies.cc0000664000175000017500000005672215161702250024077 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// Tests for FileSpaceStrategies using a mock filesystem registered via /// PathNameFactory. No real filesystem state is accessed, so the tests /// are fully deterministic. #include #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/FileSpaceStrategies.h" #include "eckit/filesystem/FileSystemSize.h" #include "eckit/filesystem/PathName.h" #include "eckit/filesystem/PathNameFactory.h" #include "eckit/testing/Test.h" using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- // FakeFileSystem — mock satisfying the BasePathNameT contract //---------------------------------------------------------------------------------------------------------------------- enum class Availability { Available, Unavailable }; enum class SizeQuery { Succeeds, Throws }; struct FakeFileSystemConfig { Availability availability; FileSystemSize size; SizeQuery sizeQuery; }; class FakeFileSystem { public: // -- Static config registry -- static std::map& configs() { static std::map m; return m; } // -- Constructors -- FakeFileSystem() : path_("/fake") {} FakeFileSystem(const std::string& p, bool /*tildeIsUserHome*/ = false) : path_(p) {} FakeFileSystem(const char* p, bool /*tildeIsUserHome*/ = false) : path_(p) {} FakeFileSystem(const FakeFileSystem& other) : path_(other.path_) {} // -- Type identifier -- static const char* type() { return "fake"; } // -- Conversion -- operator std::string() const { return path_; } friend std::ostream& operator<<(std::ostream& s, const FakeFileSystem& f) { s << f.path_; return s; } // -- Required accessors -- const std::string& path() const { return path_; } const std::string& node() const { static const std::string n = "localhost"; return n; } // -- Methods used by strategies -- bool available() const { auto it = configs().find(path_); if (it == configs().end()) { return false; } return it->second.availability == Availability::Available; } void fileSystemSize(FileSystemSize& fs) const { auto it = configs().find(path_); if (it == configs().end()) { throw Exception("FakeFileSystem: no config for " + path_); } if (it->second.sizeQuery == SizeQuery::Throws) { throw Exception("FakeFileSystem: simulated statvfs failure for " + path_); } fs = it->second.size; } // -- Stubs (not called by strategies) -- const char* localPath() const { throw NotImplemented("FakeFileSystem::localPath", Here()); } Length size() const { throw NotImplemented("FakeFileSystem::size", Here()); } time_t lastAccess() const { throw NotImplemented("FakeFileSystem::lastAccess", Here()); } time_t lastModified() const { throw NotImplemented("FakeFileSystem::lastModified", Here()); } time_t created() const { throw NotImplemented("FakeFileSystem::created", Here()); } bool isDir() const { throw NotImplemented("FakeFileSystem::isDir", Here()); } bool isLink() const { throw NotImplemented("FakeFileSystem::isLink", Here()); } bool sameAs(const FakeFileSystem&) const { throw NotImplemented("FakeFileSystem::sameAs", Here()); } FakeFileSystem mountPoint() const { throw NotImplemented("FakeFileSystem::mountPoint", Here()); } FakeFileSystem realName() const { throw NotImplemented("FakeFileSystem::realName", Here()); } bool exists() const { throw NotImplemented("FakeFileSystem::exists", Here()); } bool offsite() const { throw NotImplemented("FakeFileSystem::offsite", Here()); } std::string clusterName() const { throw NotImplemented("FakeFileSystem::clusterName", Here()); } void mkdir(short) const { throw NotImplemented("FakeFileSystem::mkdir", Here()); } void chmod(short) const { throw NotImplemented("FakeFileSystem::chmod", Here()); } void unlink(bool) const { throw NotImplemented("FakeFileSystem::unlink", Here()); } void syncParentDirectory() const { throw NotImplemented("FakeFileSystem::syncParentDirectory", Here()); } void rmdir(bool) const { throw NotImplemented("FakeFileSystem::rmdir", Here()); } void touch() const { throw NotImplemented("FakeFileSystem::touch", Here()); } void children(std::vector&, std::vector&) const { throw NotImplemented("FakeFileSystem::children", Here()); } void reserve(const Length&) const { throw NotImplemented("FakeFileSystem::reserve", Here()); } FakeFileSystem dirName() const { throw NotImplemented("FakeFileSystem::dirName", Here()); } FakeFileSystem orphanName() const { throw NotImplemented("FakeFileSystem::orphanName", Here()); } BasePathName* checkClusterNode() const { throw NotImplemented("FakeFileSystem::checkClusterNode", Here()); } FakeFileSystem fullName() const { throw NotImplemented("FakeFileSystem::fullName", Here()); } FakeFileSystem baseName(bool) const { throw NotImplemented("FakeFileSystem::baseName", Here()); } std::string extension() const { throw NotImplemented("FakeFileSystem::extension", Here()); } std::string hash(const std::string&) const { throw NotImplemented("FakeFileSystem::hash", Here()); } DataHandle* fileHandle(bool) const { throw NotImplemented("FakeFileSystem::fileHandle", Here()); } DataHandle* partHandle(const OffsetList&, const LengthList&) const { throw NotImplemented("FakeFileSystem::partHandle", Here()); } DataHandle* partHandle(const Offset&, const Length&) const { throw NotImplemented("FakeFileSystem::partHandle", Here()); } // -- Static stubs -- static void rename(const FakeFileSystem&, const FakeFileSystem&) { throw NotImplemented("FakeFileSystem::rename", Here()); } static void link(const FakeFileSystem&, const FakeFileSystem&) { throw NotImplemented("FakeFileSystem::link", Here()); } static void match(const FakeFileSystem&, std::vector&, bool) { throw NotImplemented("FakeFileSystem::match", Here()); } static FakeFileSystem unique(const FakeFileSystem&) { throw NotImplemented("FakeFileSystem::unique", Here()); } private: std::string path_; }; //---------------------------------------------------------------------------------------------------------------------- // Factory registration //---------------------------------------------------------------------------------------------------------------------- static PathNameBuilder fakeBuilder("fake"); //---------------------------------------------------------------------------------------------------------------------- // RAII scope helper — clears config before and after each test //---------------------------------------------------------------------------------------------------------------------- struct FakeFileSystemScope { FakeFileSystemScope() { FakeFileSystem::configs().clear(); } ~FakeFileSystemScope() { FakeFileSystem::configs().clear(); } }; //---------------------------------------------------------------------------------------------------------------------- // Helper functions //---------------------------------------------------------------------------------------------------------------------- PathName fakePath(const std::string& name, unsigned long long avail, unsigned long long total, Availability availability, SizeQuery sizeQuery) { FileSystemSize fs; fs.available = avail; fs.total = total; FakeFileSystem::configs()[name] = {availability, fs, sizeQuery}; return PathName("fake", name); } PathName fakePath(const std::string& name, unsigned long long avail, unsigned long long total) { return fakePath(name, avail, total, Availability::Available, SizeQuery::Succeeds); } PathName fakeUnavailablePath(const std::string& name) { return fakePath(name, 0, 0, Availability::Unavailable, SizeQuery::Succeeds); } PathName fakeFailingPath(const std::string& name) { return fakePath(name, 0, 0, Availability::Available, SizeQuery::Throws); } //---------------------------------------------------------------------------------------------------------------------- // Test cases //---------------------------------------------------------------------------------------------------------------------- CASE("leastUsed selects filesystem with most available bytes") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 100, 1000), fakePath("/fs2", 900, 1000), fakePath("/fs3", 500, 1000), }; const PathName& result = FileSpaceStrategies::leastUsed(paths); EXPECT(result == "/fs2"); } CASE("leastUsedPercent selects highest available percentage") { FakeFileSystemScope scope; // /fs1: 500/1000 = 50% free // /fs2: 200/10000 = 2% free (more absolute bytes, but lower %) // /fs3: 80/100 = 80% free std::vector paths = { fakePath("/fs1", 500, 1000), fakePath("/fs2", 200, 10000), fakePath("/fs3", 80, 100), }; const PathName& result = FileSpaceStrategies::leastUsedPercent(paths); EXPECT(result == "/fs3"); } CASE("leastUsed is deterministic") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 100, 1000), fakePath("/fs2", 900, 1000), fakePath("/fs3", 500, 1000), }; for (int i = 0; i < 20; ++i) { const PathName& result = FileSpaceStrategies::leastUsed(paths); EXPECT(result == "/fs2"); } } CASE("leastUsedPercent is deterministic") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 500, 1000), fakePath("/fs2", 80, 100), }; for (int i = 0; i < 20; ++i) { const PathName& result = FileSpaceStrategies::leastUsedPercent(paths); EXPECT(result == "/fs2"); } } CASE("Unavailable filesystems are skipped") { FakeFileSystemScope scope; std::vector paths = { fakeUnavailablePath("/fs1"), fakePath("/fs2", 100, 1000), }; const PathName& result = FileSpaceStrategies::leastUsed(paths); EXPECT(result == "/fs2"); } CASE("fileSystemSize failure is tolerated") { FakeFileSystemScope scope; std::vector paths = { fakeFailingPath("/fs1"), fakePath("/fs2", 100, 1000), }; const PathName& result = FileSpaceStrategies::leastUsed(paths); EXPECT(result == "/fs2"); } CASE("All unavailable throws Retry") { FakeFileSystemScope scope; std::vector paths = { fakeUnavailablePath("/fs1"), }; EXPECT_THROWS_AS(FileSpaceStrategies::leastUsed(paths), Retry); } CASE("Zero total size is skipped by leastUsedPercent") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 0, 0), fakePath("/fs2", 100, 1000), }; const PathName& result = FileSpaceStrategies::leastUsedPercent(paths); EXPECT(result == "/fs2"); } CASE("Filesystems over 99% used are excluded") { FakeFileSystemScope scope; // /fs1: 0/100 = 0% free → usedPercent=100 → excluded by findCandidates // /fs2: 200/1000 = 20% free → usedPercent=80 → included std::vector paths = { fakePath("/fs1", 0, 100), fakePath("/fs2", 200, 1000), }; const PathName& result = FileSpaceStrategies::pureRandom(paths); EXPECT(result == "/fs2"); } CASE("All candidates filtered falls back to leastUsed") { FakeFileSystemScope scope; // Both 100% used → findCandidates returns empty → fallback to leastUsed std::vector paths = { fakePath("/fs1", 0, 100), fakePath("/fs2", 0, 200), }; // leastUsed picks any available path (both have 0 bytes free) const PathName& result = FileSpaceStrategies::pureRandom(paths); std::string s = result.asString(); EXPECT(s == "/fs1" || s == "/fs2"); } CASE("roundRobin cycles through candidates in fixed repeating order") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 500, 1000), fakePath("/fs2", 500, 1000), fakePath("/fs3", 500, 1000), }; std::vector results; for (int i = 0; i < 6; ++i) { results.push_back(FileSpaceStrategies::roundRobin(paths).asString()); } // Consecutive calls cycle with period equal to candidate count EXPECT(results[0] == results[3]); EXPECT(results[1] == results[4]); EXPECT(results[2] == results[5]); // Consecutive calls within one cycle differ EXPECT(results[0] != results[1]); EXPECT(results[1] != results[2]); } CASE("pureRandom returns only valid candidates") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 500, 1000), fakeUnavailablePath("/fs2"), }; for (int i = 0; i < 20; ++i) { const PathName& result = FileSpaceStrategies::pureRandom(paths); EXPECT(result == "/fs1"); } } CASE("pureRandom eventually selects all candidates") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 500, 1000), fakePath("/fs2", 500, 1000), fakePath("/fs3", 500, 1000), }; // 3 uniform candidates: coupon-collector P(all seen in k) > 0.95 at k ~ 11. // Loop has no fixed bound — the test framework timeout is the failure mechanism. std::set seen; while (seen.size() < 3) { seen.insert(FileSpaceStrategies::pureRandom(paths).asString()); } // Loop exit guarantees all candidates were selected; timeout implies failure. EXPECT(true); } CASE("weightedRandom favours more available space") { FakeFileSystemScope scope; // /fs1: 900/1000 = 90% free // /fs2: 100/1000 = 10% free std::vector paths = { fakePath("/fs1", 900, 1000), fakePath("/fs2", 100, 1000), }; int fs1Count = 0; const int trials = 1000; for (int i = 0; i < trials; ++i) { const PathName& result = FileSpaceStrategies::weightedRandom(paths); if (result == "/fs1") { fs1Count++; } } // With 9:1 weight ratio, /fs1 should be selected >50% of the time EXPECT(fs1Count > trials / 2); } CASE("weightedRandomPercent favours higher free percentage") { FakeFileSystemScope scope; // /fs1: 90/100 = 90% free (less absolute space) // /fs2: 100/1000 = 10% free (more absolute space) std::vector paths = { fakePath("/fs1", 90, 100), fakePath("/fs2", 100, 1000), }; int fs1Count = 0; const int trials = 1000; for (int i = 0; i < trials; ++i) { const PathName& result = FileSpaceStrategies::weightedRandomPercent(paths); if (result == "/fs1") { fs1Count++; } } // With 90:10 percent ratio, /fs1 should be selected >50% of the time EXPECT(fs1Count > trials / 2); } CASE("binnedLeastUsed selects only from top bin") { FakeFileSystemScope scope; // /fs1 and /fs2 have similar high availability → same top bin // /fs3 has much less → lower bin // threshold = 900 - (900 - 100) / 20 = 860; /fs2 at 880 qualifies std::vector paths = { fakePath("/fs1", 900, 1000), fakePath("/fs2", 880, 1000), fakePath("/fs3", 100, 1000), }; std::set seen; for (int i = 0; i < 100; ++i) { const PathName& result = FileSpaceStrategies::binnedLeastUsed(paths); seen.insert(result.asString()); } EXPECT(seen.count("/fs1") > 0); EXPECT(seen.count("/fs2") > 0); EXPECT(seen.count("/fs3") == 0); } CASE("binnedLeastUsed with single candidate") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 500, 1000), }; const PathName& result = FileSpaceStrategies::binnedLeastUsed(paths); EXPECT(result == "/fs1"); } CASE("selectFileSystem dispatches to named strategy") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 500, 1000), fakePath("/fs2", 600, 1000), }; const std::vector strategies = { "leastUsed", "leastUsedPercent", "roundRobin", "pureRandom", "weightedRandom", "weightedRandomPercent", "binnedLeastUsed", }; for (const auto& strategy : strategies) { const PathName& result = FileSpaceStrategies::selectFileSystem(paths, strategy); std::string s = result.asString(); EXPECT(s == "/fs1" || s == "/fs2"); } } CASE("Unknown strategy name falls back to leastUsed") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 100, 1000), fakePath("/fs2", 900, 1000), }; std::string strategy = "nonexistent"; const PathName& result = FileSpaceStrategies::selectFileSystem(paths, strategy); EXPECT(result == "/fs2"); } CASE("Single filesystem always selected") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 500, 1000), }; EXPECT(FileSpaceStrategies::leastUsed(paths) == "/fs1"); EXPECT(FileSpaceStrategies::leastUsedPercent(paths) == "/fs1"); EXPECT(FileSpaceStrategies::roundRobin(paths) == "/fs1"); EXPECT(FileSpaceStrategies::pureRandom(paths) == "/fs1"); EXPECT(FileSpaceStrategies::weightedRandom(paths) == "/fs1"); EXPECT(FileSpaceStrategies::weightedRandomPercent(paths) == "/fs1"); EXPECT(FileSpaceStrategies::binnedLeastUsed(paths) == "/fs1"); } CASE("Empty vector asserts") { FakeFileSystemScope scope; std::vector paths; EXPECT_THROWS(FileSpaceStrategies::leastUsed(paths)); EXPECT_THROWS(FileSpaceStrategies::leastUsedPercent(paths)); EXPECT_THROWS(FileSpaceStrategies::roundRobin(paths)); EXPECT_THROWS(FileSpaceStrategies::pureRandom(paths)); EXPECT_THROWS(FileSpaceStrategies::weightedRandom(paths)); EXPECT_THROWS(FileSpaceStrategies::weightedRandomPercent(paths)); EXPECT_THROWS(FileSpaceStrategies::binnedLeastUsed(paths)); } //---------------------------------------------------------------------------------------------------------------------- // Coverage gap tests //---------------------------------------------------------------------------------------------------------------------- CASE("leastUsed with equal available bytes returns valid path") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 500, 1000), fakePath("/fs2", 500, 1000), }; const PathName& result = FileSpaceStrategies::leastUsed(paths); std::string s = result.asString(); EXPECT(s == "/fs1" || s == "/fs2"); } CASE("leastUsedPercent with equal percentages returns valid path") { FakeFileSystemScope scope; // Both have 50% available std::vector paths = { fakePath("/fs1", 500, 1000), fakePath("/fs2", 500, 1000), }; const PathName& result = FileSpaceStrategies::leastUsedPercent(paths); std::string s = result.asString(); EXPECT(s == "/fs1" || s == "/fs2"); } CASE("leastUsedPercent all zero total throws Retry") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 0, 0), fakePath("/fs2", 0, 0), }; EXPECT_THROWS_AS(FileSpaceStrategies::leastUsedPercent(paths), Retry); } CASE("Exactly 99% used passes candidate filter") { FakeFileSystemScope scope; // usedPercent = long(100.0 * (1000 - 10) / 1000) = long(99.0) = 99 // 99 <= 99 should pass the filter std::vector paths = { fakePath("/fs1", 10, 1000), }; const PathName& result = FileSpaceStrategies::pureRandom(paths); EXPECT(result == "/fs1"); } CASE("Zero total size skipped by findCandidates") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 0, 0), fakePath("/fs2", 500, 1000), }; // pureRandom uses findCandidates; /fs1 should be skipped for (int i = 0; i < 20; ++i) { const PathName& result = FileSpaceStrategies::pureRandom(paths); EXPECT(result == "/fs2"); } } CASE("binnedLeastUsed with all equal available selects all") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 500, 1000), fakePath("/fs2", 500, 1000), fakePath("/fs3", 500, 1000), }; // When all equal: threshold = maxAvail - 0 = maxAvail, all qualify. // 3 uniform candidates: coupon-collector P(all seen in k) > 0.95 at k ~ 11. // Loop has no fixed bound — the test framework timeout is the failure mechanism. std::set seen; while (seen.size() < 3) { seen.insert(FileSpaceStrategies::binnedLeastUsed(paths).asString()); } // Loop exit guarantees all candidates were selected; timeout implies failure. EXPECT(true); } CASE("All fileSystemSize failures throws Retry") { FakeFileSystemScope scope; std::vector paths = { fakeFailingPath("/fs1"), fakeFailingPath("/fs2"), }; EXPECT_THROWS_AS(FileSpaceStrategies::leastUsed(paths), Retry); } CASE("weightedRandom selects minority candidate") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 900, 1000), fakePath("/fs2", 100, 1000), }; // 9:1 weights: P(minority unseen after k) = 0.9^k; P(both seen) > 0.95 at k ~ 29. // Loop has no fixed bound — the test framework timeout is the failure mechanism. std::set seen; while (seen.size() < 2) { seen.insert(FileSpaceStrategies::weightedRandom(paths).asString()); } // Loop exit guarantees all candidates were selected; timeout implies failure. EXPECT(true); } CASE("weightedRandomPercent selects minority candidate") { FakeFileSystemScope scope; std::vector paths = { fakePath("/fs1", 90, 100), fakePath("/fs2", 100, 1000), }; // 9:1 weights: P(minority unseen after k) = 0.9^k; P(both seen) > 0.95 at k ~ 29. // Loop has no fixed bound — the test framework timeout is the failure mechanism. std::set seen; while (seen.size() < 2) { seen.insert(FileSpaceStrategies::weightedRandomPercent(paths).asString()); } // Loop exit guarantees all candidates were selected; timeout implies failure. EXPECT(true); } CASE("fileSystemSize failure tolerated by findCandidates-based strategy") { FakeFileSystemScope scope; std::vector paths = { fakeFailingPath("/fs1"), fakePath("/fs2", 500, 1000), }; // pureRandom goes through findCandidates, which has its own try/catch for (int i = 0; i < 20; ++i) { const PathName& result = FileSpaceStrategies::pureRandom(paths); EXPECT(result == "/fs2"); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/filesystem/CMakeLists.txt0000664000175000017500000000436215161702250021075 0ustar alastairalastair# aio system calls are know to fail on the Cray # see https://sourceware.org/bugzilla/show_bug.cgi?id=11787 option( ENABLE_ECKIT-352 "Control if ECKIT-352 tests should be enabled" ON ) ecbuild_add_test( TARGET eckit_test_filesystem_aiohandle CONDITION NOT DEFINED ENV{CRAYOS_VERSION} AND ${HAVE_AIO} SOURCES test_aiohandle.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_filesystem_asynchandle SOURCES test_asynchandle.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_filesystem_atomic_file_update SOURCES test_atomic_file_update.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_filesystem_localpathname SOURCES test_localpathname.cc LIBS eckit CONDITION ENABLE_ECKIT-352 ) ecbuild_add_test( TARGET eckit_test_filesystem_restarthandle SOURCES test_restarthandle.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_filesystem_pathexpander SOURCES test_pathexpander.cc ENVIRONMENT "CURRENT_TEST_DIR=${CMAKE_CURRENT_BINARY_DIR}" LIBS eckit ) ecbuild_add_test( TARGET eckit_test_filesystem_filemode SOURCES test_filemode.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_filesystem_uri SOURCES test_uri.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_pathname SOURCES test_pathname.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_filesystem_filespace_strategies SOURCES test_filespace_strategies.cc LIBS eckit ) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tmp/foo) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testdir/foo/1) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testdir/foo/2) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testdir/foo/2/1) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testdir/bar) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testdir/baz) eckit-2.0.7/tests/system/0000775000175000017500000000000015161702250015470 5ustar alastairalastaireckit-2.0.7/tests/system/test_system_library.cc0000664000175000017500000001065515161702250022115 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/eckit_config.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/system/Library.h" #include "eckit/system/LibraryManager.h" #include "eckit/system/Plugin.h" #include "eckit/system/ResourceUsage.h" #include "eckit/system/SystemInfo.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; using eckit::system::Library; using eckit::system::LibraryManager; using eckit::system::Plugin; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- // Issue ECKIT-520 // Just adding this plugin and registering it should trigger the registration and deregistration. // before ISSUE-520 is fixed, the deregistration may cause a segfault as the debug log channel is // destroyed before the Plugin. class TestPlugin : Plugin { public: TestPlugin() : Plugin("test-plugin") {} ~TestPlugin() { std::cout << "~TestPlugin()" << std::endl; } static const TestPlugin& instance() { static TestPlugin instance; return instance; } std::string version() const override { return "0.0.0"; } std::string gitsha1(unsigned int count) const override { return "undefined"; } }; REGISTER_LIBRARY(TestPlugin); //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_resource_usage_0") { size_t chunk = 20 * 1024 * 1024; for (size_t i = 1; i < 5; ++i) { size_t before = system::ResourceUsage().maxResidentSetSize(); void* m = ::malloc(chunk); ::memset(m, 0, chunk); size_t after = system::ResourceUsage().maxResidentSetSize(); ::free(m); Log::info() << "Memory usage " << after << std::endl; EXPECT(before <= after); } } CASE("test_eckit_system_info") { eckit::LocalPathName execPath; EXPECT_NO_THROW(execPath = eckit::system::SystemInfo::instance().executablePath()); EXPECT(std::string(execPath).size()); Log::info() << "execPath is " << execPath << std::endl; } CASE("test_eckit_system_library") { std::vector libs = LibraryManager::list(); for (std::vector::const_iterator libname = libs.begin(); libname != libs.end(); ++libname) { EXPECT_NO_THROW(LibraryManager::lookup(*libname)); const Library& lib = LibraryManager::lookup(*libname); EXPECT_NO_THROW(lib.prefixDirectory()); Log::info() << "Library " << lib.name() << " @ " << lib.prefixDirectory() << std::endl; Log::info() << lib << std::endl; } // this exercises the tilde expansion EXPECT_NO_THROW(LocalPathName("~eckit/etc").exists()); EXPECT_NO_THROW(LocalPathName("~eckit/etc/eckit/test/test.cfg").exists()); } CASE("test_libeckit") { EXPECT_NO_THROW(LibEcKit::instance().configuration()); // tests an empty configuration } #if eckit_HAVE_ECKIT_CMD CASE("test dynamic load") { EXPECT(!LibraryManager::exists("eckit_cmd")); EXPECT(LibraryManager::loadLibrary("eckit_cmd") != nullptr); EXPECT(LibraryManager::exists("eckit_cmd")); } #endif CASE("test fail dynamic load") { EXPECT(!LibraryManager::exists("fake-library")); EXPECT(LibraryManager::loadLibrary("fake-library") == nullptr); } #if eckit_HAVE_ECKIT_SQL CASE("Can load library without a eckit::Library object") { EXPECT(!LibraryManager::exists("eckit_sql")); EXPECT(LibraryManager::loadLibrary("eckit_sql") != nullptr); } #endif #if eckit_HAVE_ECKIT_MPI CASE("Fails to load a plugin without a eckit::Plugin object") { EXPECT(!LibraryManager::exists("eckit_mpi")); EXPECT_THROW_AS(LibraryManager::loadPlugin("eckit_mpi"), UnexpectedState); } #endif //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/system/CMakeLists.txt0000664000175000017500000000036715161702250020236 0ustar alastairalastair# file accessed in the test file( WRITE "${CMAKE_BINARY_DIR}/etc/eckit/test/test.cfg" "Just for testing" ) ecbuild_add_test( TARGET eckit_test_system_library SOURCES test_system_library.cc LIBS eckit ) eckit-2.0.7/tests/thread/0000775000175000017500000000000015161702250015413 5ustar alastairalastaireckit-2.0.7/tests/thread/test_mutex.cc0000664000175000017500000000303015161702250020117 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/thread/Mutex.h" #include "eckit/thread/Thread.h" #include "eckit/thread/ThreadControler.h" #include "eckit/testing/Test.h" using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- class Locker : public eckit::Thread { public: Locker(Mutex* m, bool& v) : m_(m), v_(v) {} Mutex* m_; bool& v_; void run() { v_ = m_->tryLock(); } }; CASE("Mutex Lock/Unlock") { Mutex* m; EXPECT_NO_THROW(m = new Mutex()); bool gotLock = false; EXPECT_NO_THROW(gotLock = m->tryLock()); EXPECT(gotLock); EXPECT_NO_THROW(m->unlock()); EXPECT_NO_THROW(m->lock()); ThreadControler thread(new Locker(m, gotLock), false); thread.start(); EXPECT_NO_THROW(thread.wait()); EXPECT(!gotLock); EXPECT_NO_THROW(m->unlock()); delete m; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/thread/CMakeLists.txt0000664000175000017500000000065215161702250020156 0ustar alastairalastair if( NOT ENABLE_WARNINGS ) list( APPEND suppress_warnings $<$:-h nomessage=3140> # Disable optimisation warning in debug build due to use of intrinsics ) set_source_files_properties( test_mutex.cc PROPERTIES COMPILE_OPTIONS "${suppress_warnings}" ) endif() ecbuild_add_test( TARGET eckit_test_thread_mutex SOURCES test_mutex.cc LIBS eckit ) eckit-2.0.7/tests/experimental/0000775000175000017500000000000015161702250016641 5ustar alastairalastaireckit-2.0.7/tests/experimental/singleton/0000775000175000017500000000000015161702250020643 5ustar alastairalastaireckit-2.0.7/tests/experimental/singleton/TestFactory.cc0000664000175000017500000000364315161702250023427 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include "eckit/exception/Exceptions.h" #include "TestFactory.h" struct TestFactory::PImpl { using Store = std::map; Store register_; ~PImpl() { for (Store::iterator it = register_.begin(); it != register_.end(); ++it) { delete it->second; } } void regist(const std::string& name, TestBuilder* builder) { std::cout << "registering builder [" << name << "]" << std::endl; Store::iterator itr = register_.find(name); if (itr != register_.end()) { throw std::bad_alloc(); } register_[name] = builder; } TestBuilder& get(const std::string& name) { std::cout << "get builder [" << name << "]" << std::endl; Store::iterator itr = register_.find(name); if (itr == register_.end()) { throw std::bad_alloc(); } ASSERT(itr->second); return *(itr->second); } }; //---------------------------------------------------------------------------------------------------------------------- TestFactory::TestFactory() : pimpl_(new TestFactory::PImpl()) {} TestFactory& TestFactory::instance() { // autolock mutex static TestFactory factory; return factory; } void TestFactory::regist(const std::string& name, TestBuilder* builder) { pimpl_->regist(name, builder); } TestBuilder& TestFactory::get(const std::string& name) { return pimpl_->get(name); } eckit-2.0.7/tests/experimental/singleton/test_singleton.cc0000664000175000017500000000117615161702250024220 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "TestFactory.h" int main(int argc, char** argv) { TestBuilder& b1 = TestFactory::instance().get("b1"); TestBuilder& b2 = TestFactory::instance().get("b2"); b1.build(); b2.build(); return 0; } eckit-2.0.7/tests/experimental/singleton/TestBuilder1.cc0000664000175000017500000000133115161702250023457 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "TestFactory.h" class TestBuilder1 : public TestBuilder { public: void build() { std::cout << "building a b1" << std::endl; } }; //---------------------------------------------------------------------------------------------------------------------- AutoRegistBuilder builder1("b1"); eckit-2.0.7/tests/experimental/singleton/TestBuilder2.cc0000664000175000017500000000133115161702250023460 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "TestFactory.h" class TestBuilder2 : public TestBuilder { public: void build() { std::cout << "building a b2" << std::endl; } }; //---------------------------------------------------------------------------------------------------------------------- AutoRegistBuilder builder2("b2"); eckit-2.0.7/tests/experimental/singleton/TestFactory.h0000664000175000017500000000213115161702250023260 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef TestFactory_h #define TestFactory_h #include #include #include class TestBuilder { public: virtual ~TestBuilder() {} virtual void build() = 0; }; class TestFactory { public: // methods TestFactory(); static TestFactory& instance(); void regist(const std::string&, TestBuilder*); TestBuilder& get(const std::string&); private: struct PImpl; std::unique_ptr pimpl_; }; template class AutoRegistBuilder { public: AutoRegistBuilder(const std::string& name) { std::cout << "auto register [" << name << "]" << std::endl; TestFactory::instance().regist(name, new T()); } }; #endif /* TestFactory_h */ eckit-2.0.7/tests/experimental/singleton/CMakeLists.txt0000664000175000017500000000173615161702250023412 0ustar alastairalastairecbuild_append_to_rpath( "./" ) ecbuild_add_library( TARGET eckit_test_experimental_singleton_factory TYPE SHARED NOINSTALL SOURCES TestFactory.cc TestFactory.h PUBLIC_LIBS eckit ) ecbuild_add_library( TARGET eckit_test_experimental_singleton_builder1 TYPE SHARED NOINSTALL SOURCES TestBuilder1.cc PUBLIC_LIBS eckit_test_experimental_singleton_factory ) ecbuild_add_library( TARGET eckit_test_experimental_singleton_builder2 TYPE SHARED NOINSTALL SOURCES TestBuilder2.cc PUBLIC_LIBS eckit_test_experimental_singleton_factory ) ecbuild_add_test( TARGET eckit_test_experimental_singleton_singleton SOURCES test_singleton.cc LIBS eckit_test_experimental_singleton_builder1 eckit_test_experimental_singleton_builder2 ) eckit-2.0.7/tests/experimental/CMakeLists.txt0000664000175000017500000000063115161702250021401 0ustar alastairalastair# (C) Copyright 1996- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. add_subdirectory( singleton ) eckit-2.0.7/tests/runtime/0000775000175000017500000000000015161702250015627 5ustar alastairalastaireckit-2.0.7/tests/runtime/test_telemetry.cc0000664000175000017500000000645315161702250021217 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/Resource.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/filesystem/TmpFile.h" #include "eckit/io/DataHandle.h" #include "eckit/io/FileHandle.h" #include "eckit/runtime/Telemetry.h" #include "eckit/runtime/Tool.h" #include "eckit/testing/Test.h" #include "eckit/types/Types.h" using namespace std; using namespace eckit; using namespace eckit::runtime; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- class Tester { public: Tester() { format(std::cout, Log::fullFormat); std::string base = Resource("$TMPDIR", "/tmp"); config_ = PathName::unique(base + "/telemetry.yaml"); } ~Tester() { config_.unlink(); } void setConfig(const std::string& cfg) { std::unique_ptr dh(config_.fileHandle()); dh->openForWrite(0); dh->write(cfg.data(), cfg.size()); dh->close(); ::setenv("TELEMETRY_CONFIG_FILE", config_.asString().c_str(), 1); } PathName config_; }; //---------------------------------------------------------------------------------------------------------------------- CASE("Telemetry standard configuration") { Tester tester; const char* config_str = R"XXX( --- service_type: myServiceType service_name: myServiceName service_groups: - group1 - group2 servers: - host: localhost port: 30302 )XXX"; eckit::testing::SetEnv env("TELEMETRY_CONFIG", config_str); SECTION("All messages sent") { std::string msg; EXPECT_NO_THROW(msg = Telemetry::report(Report::APPSTART)); { // std::cout << msg << std::endl; YAMLConfiguration cfg(msg); EXPECT(cfg.getInt("version") == 1); EXPECT(cfg.getString("type") == "appstart"); EXPECT(cfg.getString("service_type") == "myServiceType"); EXPECT(cfg.getString("service_name") == "myServiceName"); EXPECT(cfg.getString("service_name") == "myServiceName"); std::vector groups = {"group1", "group2"}; EXPECT(cfg.getStringVector("service_groups") == groups); } EXPECT_EQUAL(Telemetry::countSent(), 1); EXPECT_NO_THROW(Telemetry::report(Report::INFO)); EXPECT_NO_THROW(Telemetry::report(Report::METER)); EXPECT_NO_THROW(Telemetry::report(Report::KEEPALIVE)); EXPECT_EQUAL(Telemetry::countSent(), 4); EXPECT_NO_THROW(Telemetry::report(Report::COUNTER)); EXPECT_NO_THROW(Telemetry::report(Report::APPSTOP)); EXPECT_EQUAL(Telemetry::countSent(), 6); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/runtime/test_metrics.cc0000664000175000017500000002247515161702250020655 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/log/Log.h" #include "eckit/testing/Test.h" #include "eckit/runtime/Metrics.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- void expect(std::string expected, const eckit::CollectMetrics& metrics) { std::ostringstream oss; oss << metrics; expected.erase(std::remove_if(expected.begin(), expected.end(), [](unsigned char c) { return std::isspace(c); }), expected.end()); Log::info() << "Metrics: " << oss.str() << std::endl << " expected: " << expected << std::endl; EXPECT(oss.str().find(expected) != std::string::npos); } static auto values = R"(,"test": { "bool": true, "double": 3.14159, "int": -42, "length": 1024, "llong": -4200000000000, "long": -4200000000, "offset": 2048, "uint": 42, "ullong": 4200000000000, "ulong": 4200000000}})"; static auto valuesLists = R"(,"test": { "bool": true, "double": 3.14159, "int": -42, "length": 1024, "llong": -4200000000000, "long": -4200000000, "offset": 2048, "string_map": {"first": 1, "second": 2, "third": 3}, "string_set": ["alpha", "beta", "gamma"], "string_vector": ["one", "two", "three"], "uint": 42, "ullong": 4200000000000, "ulong": 4200000000}})"; CASE("test_default") { eckit::CollectMetrics metrics; Metrics::set("test.bool", true); expect(",\"test\":{\"bool\":true}}", metrics); Metrics::set("test.int", -42); Metrics::set("test.uint", 42u); Metrics::set("test.long", -4200000000l); Metrics::set("test.ulong", 4200000000ul); Metrics::set("test.llong", -4200000000000ll); Metrics::set("test.ullong", 4200000000000ull); Metrics::set("test.double", 3.14159); Metrics::set("test.length", Length(1024)); Metrics::set("test.offset", Offset(2048)); expect(values, metrics); Metrics::set("test.string_vector", std::vector{"one", "two", "three"}); Metrics::set("test.string_set", std::set{"alpha", "beta", "gamma"}); Metrics::set("test.string_map", std::map{{"first", 1}, {"second", 2}, {"third", 3}}); expect(valuesLists, metrics); Metrics::set("test.string_set", std::set{"gamma", "alpha", "beta"}); expect(valuesLists, metrics); Metrics::set("test.string_vector", std::vector{"three", "one", "two"}); expect( R"(,"test": { "bool": true, "double": 3.14159, "int": -42, "length": 1024, "llong": -4200000000000, "long": -4200000000, "offset": 2048, "string_map": {"first": 1, "second": 2, "third": 3}, "string_set": ["alpha", "beta", "gamma"], "string_vector": ["three", "one", "two"], "uint": 42, "ullong": 4200000000000, "ulong": 4200000000}})", metrics); } CASE("test_prefix") { eckit::CollectMetrics metrics; eckit::MetricsPrefix pfx("test"); Metrics::set("bool", true); expect(R"(,"test": {"bool": true}})", metrics); Metrics::set("int", -42); Metrics::set("uint", 42u); Metrics::set("long", -4200000000l); Metrics::set("ulong", 4200000000ul); Metrics::set("llong", -4200000000000ll); Metrics::set("ullong", 4200000000000ull); Metrics::set("double", 3.14159); Metrics::set("length", Length(1024)); Metrics::set("offset", Offset(2048)); expect(values, metrics); Metrics::set("string_vector", std::vector{"one", "two", "three"}); Metrics::set("string_set", std::set{"alpha", "beta", "gamma"}); Metrics::set("string_map", std::map{{"first", 1}, {"second", 2}, {"third", 3}}); expect(valuesLists, metrics); Metrics::set("string_set", std::set{"gamma", "alpha", "beta"}); expect(valuesLists, metrics); } CASE("test_nested_prefixes") { eckit::CollectMetrics metrics; eckit::MetricsPrefix pfx("test"); { eckit::MetricsPrefix pfx2("nested"); Metrics::set("bool", true); expect(R"(,"test": {"nested": {"bool": true}}})", metrics); Metrics::set("third.int", -42); Metrics::set("uint", 42u); Metrics::set("long", -4200000000l); Metrics::set("ulong", 4200000000ul); Metrics::set("llong", -4200000000000ll); Metrics::set("ullong", 4200000000000ull); Metrics::set("double", 3.14159); Metrics::set("length", Length(1024)); Metrics::set("offset", Offset(2048)); expect(R"(,"test": { "nested": { "bool": true, "double": 3.14159, "length": 1024, "llong": -4200000000000, "long": -4200000000, "offset": 2048, "third": { "int": -42}, "uint": 42, "ullong": 4200000000000, "ulong": 4200000000}}})", metrics); } } static auto g1_value = R"(, "group": [ {"int": 1} ]})"; static auto g2_value = R"(, "group": [ {"int": 1}, {"int": 2} ]})"; CASE("test_group1") { eckit::CollectMetrics metrics; { eckit::MetricsGroup grp("group"); Metrics::set("int", 1); } expect(g1_value, metrics); } CASE("test_group2") { eckit::CollectMetrics metrics; { eckit::MetricsGroup grp("group"); Metrics::set("int", 1); eckit::MetricsGroupItem gi; Metrics::set("int", 2); } expect(g2_value, metrics); } CASE("test_group3") { eckit::CollectMetrics metrics; { eckit::MetricsGroup grp("group"); Metrics::set("int", 1); } expect(g1_value, metrics); { eckit::MetricsGroup grp("group"); eckit::MetricsGroupItem gi1; Metrics::set("int", 2); eckit::MetricsGroupItem gi2; } expect(g2_value, metrics); } CASE("test_group4") { eckit::CollectMetrics metrics; { eckit::MetricsGroup grp("group"); for (int i = 1; i < 4; ++i) { eckit::MetricsGroupItem gi; Metrics::set("int", i); } } expect(R"(, "group": [ {"int": 1}, {"int": 2}, {"int": 3} ]})", metrics); Metrics::set("int", -42); expect(R"(, "group": [ {"int": 1}, {"int": 2}, {"int": 3} ], "int": -42})", metrics); } CASE("test_prefix_and_group") { eckit::CollectMetrics metrics; eckit::MetricsPrefix p2("p1"); { eckit::MetricsGroup g1("g1"); for (int i = 1; i < 4; ++i) { eckit::MetricsGroupItem gi; Metrics::set("int", i); } } expect(R"(, "p1": { "g1": [ {"int": 1}, {"int": 2}, {"int": 3} ]} })", metrics); } CASE("test_group_and_prefix") { eckit::CollectMetrics metrics; { eckit::MetricsGroup g1("g1"); for (int i = 1; i < 4; ++i) { eckit::MetricsGroupItem gi; { eckit::MetricsPrefix p2("p1"); Metrics::set("int", i); } } } expect(R"(,"g1":[ {"p1": {"int":1}}, {"p1": {"int":2}}, {"p1": {"int":3}}]})", metrics); } CASE("test_nested_groups") { eckit::CollectMetrics metrics; { eckit::MetricsGroup g1("g1"); for (int i = 1; i < 4; ++i) { eckit::MetricsGroupItem g1i; eckit::MetricsPrefix pfx("p1"); eckit::MetricsGroup g2("g2"); for (int j = i; j < 4; ++j) { eckit::MetricsGroupItem g2i; Metrics::set("int", j); } } } expect(R"(,"g1":[ {"p1": {"g2":[ {"int":1},{"int":2},{"int":3}]}}, {"p1": {"g2":[ {"int":2},{"int":3}]}}, {"p1": {"g2":[ {"int":3}]}}]})", metrics); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/runtime/test_producer.cc0000664000175000017500000000316615161702250021026 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/runtime/ProducerConsumer.h" #include "eckit/runtime/Tool.h" #include "eckit/utils/Translator.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- struct C : public Consumer { virtual void consume(string& s) { Log::info() << "Consume " << s << std::endl; ::usleep(10000); } }; struct P : public Producer { int count_; virtual bool done() { return count_ <= 0; } virtual void produce(string& s) { Log::info() << "Produce " << count_ << std::endl; ::usleep(count_ * 10000); EXPECT(count_); s = string("Hello, world! ") + Translator()(count_); count_--; } P() : count_(5) {} }; CASE("test_producer") { P p; C c; ProducerConsumer pc; pc.execute(p, c); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/runtime/test_context.cc0000664000175000017500000000250315161702250020661 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_default") { /* log before context build */ Log::info() << "logging before calling Context" << std::endl; /* setting context another time */ Log::info() << "logging after resetting behavior" << std::endl; Log::debug() << "logging after resetting behavior" << std::endl; Log::warning() << "logging after resetting behavior" << std::endl; Log::error() << "logging after resetting behavior" << std::endl; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/runtime/test_taskinfo.cc0000664000175000017500000001773215161702250021025 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include #include #include #include #include "eckit/memory/Padded.h" #include "eckit/runtime/Main.h" #include "eckit/runtime/TaskInfo.h" #include "eckit/testing/Test.h" namespace eckit::test { CASE("TaskInfo correctly initializes all members on placement new") { // Ensure we TaskInfo is a wrapper around Padded with no // additional fields so that we can test Padded as proxy of TaskInfo static_assert(sizeof(Padded) == sizeof(TaskInfo)); static_assert(std::is_trivially_copyable_v>); const auto test_start_time = ::time(nullptr); auto getTimeOfDayNow = []() { ::timeval t{}; ::gettimeofday(&t, nullptr); return t; }; const auto test_start_time_of_day = getTimeOfDayNow(); auto timevalCmp = [](const struct timeval& a, const struct timeval& b) { if (a.tv_sec < b.tv_sec) return -1; if (a.tv_sec > b.tv_sec) return 1; if (a.tv_usec < b.tv_usec) return -1; if (a.tv_usec > b.tv_usec) return 1; return 0; }; alignas(TaskInfo) std::vector buffer(sizeof(TaskInfo), 0xAF); // We *could* just reinterpret_cast TaskInfo into Padded but this is // undefined behaviour because Padded is not standart layout altough it // is the same layout. Hence we memcpy auto sut = new (buffer.data()) TaskInfo(); Padded info{}; ::memcpy(&info, sut, sizeof(info)); EXPECT_EQUAL(info.busy_, true); EXPECT_EQUAL(info.thread_, pthread_self()); EXPECT_EQUAL(info.pid_, getpid()); // start is initialized to now on construction, once we reach this line // we no longer know when this was, so we just chcek for later than test start // but earler than now. EXPECT(info.start_ >= test_start_time); EXPECT(info.start_ <= ::time(nullptr)); // last_ and check_ are initialized to now on construction, once we reach this line // we no longer know when this was, so we just chcek for later than test start // but earler than now. EXPECT(info.last_ >= test_start_time); EXPECT(info.last_ <= ::time(nullptr)); EXPECT(info.check_ >= test_start_time); EXPECT(info.check_ <= ::time(nullptr)); EXPECT(info.show_); EXPECT_EQUAL(info.late_, 0); EXPECT(std::all_of(info.buffer_, info.buffer_ + Info::size_, [](auto v) { return v == 0x00; })); EXPECT_EQUAL(info.pos_, 0); std::array expected_name{}; ::strncpy(expected_name.begin(), Main::instance().displayName().c_str(), expected_name.size() - 1); EXPECT_EQUAL(::memcmp(info.name_, expected_name.begin(), expected_name.size()), 0); std::array expected_kind{}; ::strncpy(expected_kind.begin(), Main::instance().name().c_str(), expected_kind.size() - 1); EXPECT_EQUAL(::memcmp(info.kind_, expected_kind.begin(), expected_kind.size()), 0); std::array expected_status{}; ::strncpy(expected_status.begin(), "Starting", expected_status.size() - 1); EXPECT_EQUAL(::memcmp(info.status_, expected_status.begin(), expected_status.size()), 0); std::array expected_application{}; ::strncpy(expected_application.begin(), Main::instance().name().c_str(), expected_application.size() - 1); EXPECT_EQUAL(::memcmp(info.application_, expected_application.begin(), expected_application.size()), 0); EXPECT_EQUAL(info.progress_.min_, 0); EXPECT_EQUAL(info.progress_.max_, 0); EXPECT_EQUAL(info.progress_.val_, 0); EXPECT(std::all_of(info.progress_.name_, info.progress_.name_ + sizeof(info.progress_.name_), [](auto v) { return v == 0x00; })); EXPECT_EQUAL(info.progress_.rate_, 0); EXPECT_EQUAL(info.progress_.speed_, 0); // progress_.start and progress_.last_ are initialized to time of day now // on construction, once we reach this line we no longer know when this // was, so we just chcek for later than test start but earler than now. EXPECT(timevalCmp(info.progress_.start_, test_start_time_of_day) >= 0); EXPECT(timevalCmp(info.progress_.start_, getTimeOfDayNow()) <= 0); EXPECT(timevalCmp(info.progress_.last_, test_start_time_of_day) >= 0); EXPECT(timevalCmp(info.progress_.last_, getTimeOfDayNow()) <= 0); EXPECT_EQUAL(info.taskID_, 0); EXPECT_EQUAL(info.stop_, false); EXPECT_EQUAL(info.abort_, false); EXPECT_EQUAL(info.stoppable_, true); EXPECT_EQUAL(info.stopped_, false); EXPECT_EQUAL(info.canceled_, false); EXPECT_EQUAL(info.exception_, false); EXPECT(std::all_of(info.cancelMsg_, info.cancelMsg_ + sizeof(info.cancelMsg_), [](auto v) { return v == 0x00; })); EXPECT_EQUAL(info.config_, 0); EXPECT_EQUAL(info.resource_, 0); EXPECT_EQUAL(info.parent_, -1); EXPECT_EQUAL(info.depth_, 0); EXPECT_EQUAL(info.state_, ' '); EXPECT_EQUAL(info.port_, 0); EXPECT(std::all_of(info.host_, info.host_ + sizeof(info.host_), [](auto v) { return v == 0x00; })); EXPECT(std::all_of(info.message_, info.message_ + sizeof(info.message_), [](auto v) { return v == 0x00; })); } CASE("Info struct binary compatibility") { // The Info struct is persisted to disk via MappedArray and SharedMemArray. // Processes read each other's monitor files by memory-mapping them and // interpreting raw bytes as Info. If the struct layout changes (field sizes, // offsets, or total size), existing files written by older binaries become // unreadable or silently corrupt. These tests ensure that any layout-breaking // change is caught immediately. Info info{}; // Struct sizes EXPECT_EQUAL(sizeof(Info), 11248u); EXPECT_EQUAL(sizeof(Info::Progress), 152u); EXPECT_EQUAL(sizeof(TaskInfo), 12288u); EXPECT_EQUAL(sizeof(Padded), 12288u); // Circular log buffer constant EXPECT_EQUAL(Info::size_, 10240); // Individual field sizes EXPECT_EQUAL(sizeof(info.busy_), 1u); EXPECT_EQUAL(sizeof(info.thread_), 8u); EXPECT_EQUAL(sizeof(info.pid_), 4u); EXPECT_EQUAL(sizeof(info.start_), 8u); EXPECT_EQUAL(sizeof(info.last_), 8u); EXPECT_EQUAL(sizeof(info.check_), 8u); EXPECT_EQUAL(sizeof(info.show_), 1u); EXPECT_EQUAL(sizeof(info.late_), 8u); EXPECT_EQUAL(sizeof(info.buffer_), 10240u); EXPECT_EQUAL(sizeof(info.pos_), 8u); EXPECT_EQUAL(sizeof(info.name_), 80u); EXPECT_EQUAL(sizeof(info.kind_), 80u); EXPECT_EQUAL(sizeof(info.status_), 256u); EXPECT_EQUAL(sizeof(info.application_), 80u); EXPECT_EQUAL(sizeof(info.progress_), 152u); EXPECT_EQUAL(sizeof(info.progress_.name_), 80u); EXPECT_EQUAL(sizeof(info.taskID_), 8u); EXPECT_EQUAL(sizeof(info.cancelMsg_), 80u); EXPECT_EQUAL(sizeof(info.host_), 80u); EXPECT_EQUAL(sizeof(info.message_), 80u); EXPECT_EQUAL(sizeof(info.stop_), 1u); EXPECT_EQUAL(sizeof(info.abort_), 1u); EXPECT_EQUAL(sizeof(info.stoppable_), 1u); EXPECT_EQUAL(sizeof(info.stopped_), 1u); EXPECT_EQUAL(sizeof(info.canceled_), 1u); EXPECT_EQUAL(sizeof(info.exception_), 1u); EXPECT_EQUAL(sizeof(info.config_), 4u); EXPECT_EQUAL(sizeof(info.resource_), 1u); EXPECT_EQUAL(sizeof(info.parent_), 8u); EXPECT_EQUAL(sizeof(info.depth_), 8u); EXPECT_EQUAL(sizeof(info.state_), 1u); EXPECT_EQUAL(sizeof(info.port_), 4u); // Type traits EXPECT(std::is_standard_layout_v); EXPECT(std::is_trivially_copyable_v); } } // namespace eckit::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/runtime/CMakeLists.txt0000664000175000017500000000132015161702250020363 0ustar alastairalastairecbuild_add_test( TARGET eckit_test_runtime_producer SOURCES test_producer.cc ENVIRONMENT _TEST_ECKIT_HOME=/tmp/$ENV{USER} LIBS eckit ) ecbuild_add_test( TARGET eckit_test_runtime_telemetry SOURCES test_telemetry.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_runtime_context SOURCES test_context.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_runtime_taskinfo SOURCES test_taskinfo.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_runtime_metrics SOURCES test_metrics.cc LIBS eckit ) eckit-2.0.7/tests/parser/0000775000175000017500000000000015161702250015440 5ustar alastairalastaireckit-2.0.7/tests/parser/2.6.yaml0000664000175000017500000000012015161702250016622 0ustar alastairalastairMark McGwire: {hr: 65, avg: 0.278} Sammy Sosa: { hr: 63, avg: 0.288 } eckit-2.0.7/tests/parser/test_stream_parser.cc0000664000175000017500000000642315161702250021662 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/parser/StreamParser.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_parser_stream_next") { std::stringstream ss; ss << "1234567890abcdefgh"; eckit::StreamParser parser(ss); EXPECT(parser.next() == '1'); EXPECT(parser.next() == '2'); for (int i = 0; i < 15; i++) { parser.next(); } EXPECT(parser.next() == 'h'); } CASE("test_eckit_parser_stream_next_spaces") { std::stringstream ss; ss << "1 3 6 ab def h"; eckit::StreamParser parser(ss); EXPECT(parser.next() == '1'); EXPECT(parser.next() == '3'); // n.b. by default spaces=false EXPECT(parser.next(true) == ' '); EXPECT(parser.next(true) == ' '); EXPECT(parser.next(true) == '6'); EXPECT(parser.next(true) == ' '); EXPECT(parser.next(false) == 'a'); } CASE("test_eckit_parser_stream_next_eof") { // cf. commit: 7224cc431 std::stringstream ss; ss << "12"; eckit::StreamParser parser(ss); EXPECT(parser.next() == '1'); EXPECT(parser.next() == '2'); EXPECT_THROWS_AS(parser.next(), StreamParser::Error); } CASE("test_eckit_parser_stream_peek") { std::stringstream ss; ss << " 2 5"; eckit::StreamParser parser(ss); EXPECT(parser.peek() == '2'); // n.b. by default spaces=false EXPECT(parser.peek() == '2'); EXPECT(parser.next() == '2'); EXPECT(parser.peek(true) == ' '); EXPECT(parser.peek(false) == '5'); EXPECT(parser.peek(true) == '5'); EXPECT(parser.next(true) == '5'); // And check eof behaviour EXPECT(parser.peek() == 0); } CASE("test_eckit_parser_stream_consume_char") { std::stringstream ss; ss << " 2 567 "; eckit::StreamParser parser(ss); // n.b. by default the StreamParser ignores spaces. parser.consume('2'); parser.consume('5'); // Check that it throws on mismatch EXPECT_THROWS_AS(parser.consume('7'), StreamParser::Error); // 6 != 7 parser.consume('7'); // Check that it throws on eof EXPECT_THROWS_AS(parser.consume('8'), StreamParser::Error); } CASE("test_eckit_parser_stream_consume_string") { std::stringstream ss; ss << " 234 789 0a"; eckit::StreamParser parser(ss); // n.b. by default the StreamParser ignores spaces. parser.consume("234"); // Check that it throws on mismatched contents EXPECT_THROWS_AS(parser.consume("779"), StreamParser::Error); // Check that it throws on eof EXPECT_THROWS_AS(parser.consume("0ab"), StreamParser::Error); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/parser/2.16.yaml0000664000175000017500000000021315161702250016706 0ustar alastairalastairname: Mark McGwire accomplishment: > Mark set a major league home run record in 1998. stats: | 65 Home Runs 0.278 Batting Average eckit-2.0.7/tests/parser/2.17.yaml0000664000175000017500000000026115161702250016712 0ustar alastairalastairunicode: "Sosa did fine.\u263A" control: "\b1998\t1999\t2000\n" hexesc: "\x13\x10 is \r\n" single: '"Howdy!" he cried.' quoted: ' # not a ''comment''.' tie-fighter: '|\-*-/|' eckit-2.0.7/tests/parser/2.10.yaml0000664000175000017500000000017715161702250016711 0ustar alastairalastair--- hr: - Mark McGwire # Following node labeled SS - &SS Sammy Sosa rbi: - *SS # Subsequent occurrence - Ken Griffey eckit-2.0.7/tests/parser/2.1.yaml0000664000175000017500000000005215161702250016621 0ustar alastairalastair- Mark McGwire - Sammy Sosa - Ken Griffey eckit-2.0.7/tests/parser/2.4.yaml0000664000175000017500000000013615161702250016627 0ustar alastairalastair- name: Mark McGwire hr: 65 avg: 0.278 - name: Sammy Sosa hr: 63 avg: 0.288 eckit-2.0.7/tests/parser/2.11.yaml0000664000175000017500000000021615161702250016704 0ustar alastairalastair? - Detroit Tigers - Chicago cubs : - 2001-07-23 ? [ New York Yankees, Atlanta Braves ] : [ 2001-07-02, 2001-08-12, 2001-08-14 ] eckit-2.0.7/tests/parser/2.5.yaml0000664000175000017500000000012415161702250016625 0ustar alastairalastair- [name , hr, avg ] - [Mark McGwire, 65, 0.278] - [Sammy Sosa , 63, 0.288] eckit-2.0.7/tests/parser/test_json_metadata.cc0000664000175000017500000001152515161702250021623 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/io/Buffer.h" #include "eckit/parser/JSONMetadata.h" #include "eckit/types/FloatCompare.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::types; using namespace eckit::testing; namespace eckit { namespace test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_json_metadata_params") { std::string json_str = "{\"key1\": 1, \"key2\": 99.5, \"key3\": []}"; Buffer buf(json_str.c_str(), json_str.length()); JSONMetadata md(buf); std::vector params(md.keywords()); EXPECT(params.size() == 3); EXPECT(std::find(params.begin(), params.end(), "key1") != params.end()); EXPECT(std::find(params.begin(), params.end(), "key2") != params.end()); EXPECT(std::find(params.begin(), params.end(), "key3") != params.end()); } CASE("test_eckit_json_metadata_params_nondict") { // If a valid json is supplied that is not an object/dict, there should be no // internal parameters to find std::string json_str = "[1, 2, 44.5]"; Buffer buf(json_str.c_str(), json_str.length()); JSONMetadata md(buf); std::vector params(md.keywords()); EXPECT(params.size() == 0); } CASE("test_eckit_json_metadata_has") { std::string json_str = "{\"key1\": 1, \"key2\": 99.5, \"key3\": []}"; Buffer buf(json_str.c_str(), json_str.length()); JSONMetadata md(buf); EXPECT(md.has("key1")); EXPECT(md.has("key2")); EXPECT(md.has("key3")); EXPECT(!md.has("keyunused")); } CASE("test_eckit_json_metadata_has_nondict") { // If a valid json is supplied that is not an object/dict, there should be no // internal parameters to find std::string json_str = "[1, 2, 44.5]"; Buffer buf(json_str.c_str(), json_str.length()); JSONMetadata md(buf); EXPECT(!md.has("key1")); } CASE("test_eckit_json_metadata_get_string") { std::string json_str = "{\"key1\": 1, \"key2\": \"testing string\", \"key3\": []}"; Buffer buf(json_str.c_str(), json_str.length()); JSONMetadata md(buf); std::string str_tmp; md.get("key2", str_tmp); // Gets value correctly EXPECT(str_tmp == "testing string"); // Throws on incorrect type EXPECT_THROWS_AS(md.get("key3", str_tmp), BadCast); // Throws on missing key EXPECT_THROWS_AS(md.get("keyunused", str_tmp), OutOfRange); // Throws on non-dictionary root-value std::string json_str2 = "[1234]"; Buffer buf2(json_str2.c_str(), json_str2.length()); JSONMetadata md2(buf2); EXPECT_THROWS_AS(md2.get("key1", str_tmp), AssertionFailed); } CASE("test_eckit_json_metadata_get_double") { std::string json_str = "{\"key1\": 123.45, \"key2\": \"testing string\", \"key3\": [], \"key4\": 123}"; Buffer buf(json_str.c_str(), json_str.length()); JSONMetadata md(buf); double dbl_tmp; md.get("key1", dbl_tmp); EXPECT(is_approximately_equal(dbl_tmp, 123.45, 1.0e-6)); md.get("key4", dbl_tmp); EXPECT(is_approximately_equal(dbl_tmp, static_cast(123), 1.0e-6)); // Throws on incorrect type EXPECT_THROWS_AS(md.get("key3", dbl_tmp), BadCast); // Throws on missing key EXPECT_THROWS_AS(md.get("keyunused", dbl_tmp), OutOfRange); // Throws on non-dictionary root-value std::string json_str2 = "[1234]"; Buffer buf2(json_str2.c_str(), json_str2.length()); JSONMetadata md2(buf2); EXPECT_THROWS_AS(md2.get("key1", dbl_tmp), AssertionFailed); } CASE("test_eckit_json_metadata_get_long") { std::string json_str = "{\"key1\": 123.45, \"key2\": \"testing string\", \"key3\": [], \"key4\": 123}"; Buffer buf(json_str.c_str(), json_str.length()); JSONMetadata md(buf); long long_tmp; md.get("key4", long_tmp); EXPECT(long_tmp == 123); // Throws on incorrect type EXPECT_THROWS_AS(md.get("key3", long_tmp), BadCast); // Throws on missing key EXPECT_THROWS_AS(md.get("keyunused", long_tmp), OutOfRange); // Throws on non-dictionary root-value std::string json_str2 = "[1234]"; Buffer buf2(json_str2.c_str(), json_str2.length()); JSONMetadata md2(buf2); EXPECT_THROWS_AS(md2.get("key1", long_tmp), AssertionFailed); } //---------------------------------------------------------------------------------------------------------------------- } // namespace test } // namespace eckit int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/parser/test_yaml.cc0000664000175000017500000005764315161702250017767 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/eckit_config.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/JSON.h" #include "eckit/parser/YAMLParser.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- static std::string toJSON(const Value& v) { std::ostringstream oss; JSON json(oss); json << v; return oss.str(); } // CASE ( "test_eckit_yaml_1" ) { // Value v = YAMLParser::decodeFile("2.1.yaml"); // std::cout << "2.1.yaml " << v << std::endl; // std::cout << toJSON(v) << std::endl; // EXPECT( v.isList() ); // EXPECT( v.size() == 3 ); // EXPECT( v[0] == "Mark McGwire" ); // EXPECT( v[1] == "Sammy Sosa" ); // EXPECT( v[2] == "Ken Griffey" ); // // EXPECT(toJSON(v) == "[\"Mark McGwire\",\"Sammy Sosa\",\"Ken Griffey\"]"); // } CASE("test_eckit_yaml_2") { Value v = YAMLParser::decodeFile("2.2.yaml"); std::cout << "2.2.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; EXPECT(v.isOrderedMap()); EXPECT(v.keys().size() == 3); EXPECT(v["hr"] == Value(65)); EXPECT(v["avg"] == Value(0.278)); EXPECT(v["rbi"] == Value(147)); } CASE("test_eckit_yaml_3") { Value v = YAMLParser::decodeFile("2.3.yaml"); std::cout << "2.3.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; EXPECT(v.isOrderedMap()); EXPECT(v.keys().size() == 2); EXPECT(v["american"].isList()); EXPECT(v["american"].size() == 3); EXPECT(v["national"].isList()); EXPECT(v["national"].size() == 3); } CASE("test_eckit_yaml_4") { Value v = YAMLParser::decodeFile("2.4.yaml"); std::cout << "2.4.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; EXPECT(v.isList()); EXPECT(v.size() == 2); EXPECT(v[0].isOrderedMap()); EXPECT(v[0].keys().size() == 3); EXPECT(v[1].isOrderedMap()); EXPECT(v[1].keys().size() == 3); } CASE("test_eckit_yaml_5") { Value v = YAMLParser::decodeFile("2.5.yaml"); std::cout << "2.5.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; EXPECT(v.isList()); EXPECT(v.size() == 3); EXPECT(v[0].isList()); EXPECT(v[0].size() == 3); EXPECT(v[1].isList()); EXPECT(v[1].size() == 3); EXPECT(v[2].isList()); EXPECT(v[2].size() == 3); } CASE("test_eckit_yaml_6") { Value v = YAMLParser::decodeFile("2.6.yaml"); std::cout << "2.6.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; EXPECT(v.isOrderedMap()); EXPECT(v.keys().size() == 2); EXPECT(v["Mark McGwire"].isOrderedMap()); EXPECT(v["Mark McGwire"].keys().size() == 2); EXPECT(v["Sammy Sosa"].isOrderedMap()); EXPECT(v["Sammy Sosa"].keys().size() == 2); } CASE("test_eckit_yaml_7") { Value v = YAMLParser::decodeFile("2.7.yaml"); std::cout << "2.7.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } CASE("test_eckit_yaml_8") { Value v = YAMLParser::decodeFile("2.8.yaml"); std::cout << "2.8.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } CASE("test_eckit_yaml_9") { Value v = YAMLParser::decodeFile("2.9.yaml"); std::cout << "2.9.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } CASE("test_eckit_yaml_10") { Value v = YAMLParser::decodeFile("2.10.yaml"); std::cout << "2.10.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } /* CASE ( "test_eckit_yaml_11" ) { Value v = YAMLParser::decodeFile("2.11.yaml"); std::cout << "2.11.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; std::cout << toJSON(v) << std::endl; } */ CASE("test_eckit_yaml_12") { Value v = YAMLParser::decodeFile("2.12.yaml"); std::cout << "2.12.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } /* CASE ( "test_eckit_yaml_13" ) { Value v = YAMLParser::decodeFile("2.13.yaml"); std::cout << "2.13.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } */ CASE("test_eckit_yaml_14") { Value v = YAMLParser::decodeFile("2.14.yaml"); std::cout << "2.14.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; std::cout << "----------" << std::endl; std::cout << v << std::endl; std::cout << "----------" << std::endl; EXPECT(v == "Mark McGwire's year was crippled by a knee injury."); } CASE("test_eckit_yaml_15") { Value v = YAMLParser::decodeFile("2.15.yaml"); std::cout << "2.15.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; std::cout << "----------" << std::endl; std::cout << v << std::endl; std::cout << "----------" << std::endl; EXPECT(v == "Sammy Sosa completed another fine season with great stats.\n\n 63 Home Runs\n 0.288 Batting " "Average\n\nWhat a year!\n"); } CASE("test_eckit_yaml_16") { Value v = YAMLParser::decodeFile("2.16.yaml"); std::cout << "2.16.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } // CASE ( "test_eckit_yaml_17" ) { // Value v = YAMLParser::decodeFile("2.17.yaml"); // std::cout << "2.17.yaml " << v << std::endl; // std::cout << toJSON(v) << std::endl; // } CASE("test_eckit_yaml_18") { Value v = YAMLParser::decodeFile("2.18.yaml"); std::cout << "2.18.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } CASE("test_eckit_yaml_19") { Value v = YAMLParser::decodeFile("2.19.yaml"); std::cout << "2.19.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; EXPECT(v["base8"] == Value(13579)); EXPECT(v["base10"] == Value(13579)); EXPECT(v["base16"] == Value(13579)); EXPECT(v["base_8"] == Value(24680)); EXPECT(v["base_10"] == Value(24680)); EXPECT(v["base_16"] == Value(24680)); EXPECT(v["zerobase10"] == Value(13579)); EXPECT(v["plus-base10"] == Value(13579)); EXPECT(v["minus-base10"] == Value(-13579)); EXPECT(v["str-base8"] == Value("0o32413")); EXPECT(v["str-base10"] == Value("13579")); EXPECT(v["str-base16"] == Value("0x350b")); EXPECT(v["zero"] == Value(0)); EXPECT(v["minus-zero"] == Value(0)); EXPECT(v["plus-zero"] == Value(0)); } CASE("test_eckit_yaml_20") { Value v = YAMLParser::decodeFile("2.20.yaml"); std::cout << "2.20.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; EXPECT(fabs((double)v["canonical"] - 1230.15) < 1.0e-7); EXPECT(fabs((double)v["exponential"] - 1230.15) < 1.0e-7); EXPECT(fabs((double)v["fixed"] - 1230.15) < 1.0e-7); EXPECT(isinf((double)v["infinity"])); EXPECT((double)v["infinity"] > 0.0); EXPECT(isinf((double)v["positive infinity"])); EXPECT((double)v["positive infinity"] > 0.0); EXPECT(isinf((double)v["negative infinity"])); EXPECT((double)v["negative infinity"] < 0.0); EXPECT(isnan((double)v["not a number"])); EXPECT(isnan((double)v["not a number lower"])); EXPECT(isnan((double)v["not a number upper"])); } CASE("test_eckit_yaml_21") { Value v = YAMLParser::decodeFile("2.21.yaml"); std::cout << "2.21.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; EXPECT(v["tilde-null"] == Value()); EXPECT(v["lower-null"] == Value()); EXPECT(v["cap-null"] == Value()); EXPECT(v["upper-null"] == Value()); EXPECT(v["lower-true"] == Value(true)); EXPECT(v["cap-true"] == Value(true)); EXPECT(v["upper-true"] == Value(true)); EXPECT(v["y-true"] == Value(true)); EXPECT(v["lower-false"] == Value(false)); EXPECT(v["cap-false"] == Value(false)); EXPECT(v["upper-false"] == Value(false)); EXPECT(v["n-false"] == Value(false)); EXPECT(v["string"] == Value("12345")); } CASE("test_eckit_yaml_21_extra") { // Associated with ECKIT-260: // Case test_eckit_yaml_21 caused problems, because "true"/"false" is interpreted as a bool (rather than string) and // comparators were broken. Here we check that these keys are interpreted and compared correctly. // null: ~ // true: y // false: n // string: '12345' // Check decoding of true, false, null, etc. EXPECT(YAMLParser::decodeString("null").typeName() == "Nil"); EXPECT(YAMLParser::decodeString("false").typeName() == "Bool"); EXPECT(YAMLParser::decodeString("~").typeName() == "Nil"); EXPECT(YAMLParser::decodeString("y").typeName() == "String"); EXPECT(YAMLParser::decodeString("n").typeName() == "String"); EXPECT(YAMLParser::decodeString("string").typeName() == "String"); EXPECT(YAMLParser::decodeString("'12345'").typeName() == "String"); EXPECT(YAMLParser::decodeString("\"42\"").typeName() == "String"); EXPECT_EQUAL(YAMLParser::decodeString("42").typeName(), "Number"); EXPECT(YAMLParser::decodeString("null") == Value()); EXPECT(YAMLParser::decodeString("false") == Value(false)); EXPECT(YAMLParser::decodeString("true").typeName() == "Bool"); EXPECT(Value(true).typeName() == "Bool"); EXPECT(Value(true) == Value(true)); EXPECT(YAMLParser::decodeString("true") == YAMLParser::decodeString("true")); EXPECT(YAMLParser::decodeString("true") == Value(true)); EXPECT(YAMLParser::decodeString("false").typeName() == "Bool"); EXPECT(Value(false).typeName() == "Bool"); EXPECT(Value(false) == Value(false)); EXPECT(YAMLParser::decodeString("false") == YAMLParser::decodeString("false")); EXPECT(YAMLParser::decodeString("false") == Value(false)); EXPECT(Value(false) != Value(true)); EXPECT(YAMLParser::decodeString("false") != YAMLParser::decodeString("true")); EXPECT(YAMLParser::decodeString("false") != Value(true)); EXPECT(Value(true) != Value(false)); EXPECT(YAMLParser::decodeString("true") != YAMLParser::decodeString("false")); EXPECT(YAMLParser::decodeString("true") != Value(false)); // Check ordering and comparisons EXPECT(Value() == Value()); EXPECT(Value() < Value(true)); EXPECT(Value() < Value(false)); EXPECT(Value(false) == Value(false)); EXPECT(Value(true) == Value(true)); EXPECT(Value(false) < Value(true)); EXPECT(!(Value(true) < Value(true))); EXPECT(!(Value(false) < Value(false))); EXPECT(Value("Something") == Value("Something")); } CASE("test_eckit_yaml_22") { Value v = YAMLParser::decodeFile("2.22.yaml"); std::cout << "2.22.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } // CASE ( "test_eckit_yaml_23" ) { // Value v = YAMLParser::decodeFile("2.23.yaml"); // std::cout << "2.23.yaml " << v << std::endl; // std::cout << toJSON(v) << std::endl; // } // CASE ( "test_eckit_yaml_24" ) { // Value v = YAMLParser::decodeFile("2.24.yaml"); // std::cout << "2.24.yaml " << v << std::endl; // std::cout << toJSON(v) << std::endl; // } CASE("test_eckit_yaml_25") { Value v = YAMLParser::decodeFile("2.25.yaml"); std::cout << "2.25.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } CASE("test_eckit_yaml_26") { Value v = YAMLParser::decodeFile("2.26.yaml"); std::cout << "2.26.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } CASE("test_eckit_yaml_27") { Value v = YAMLParser::decodeFile("2.27.yaml"); std::cout << "2.27.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } // CASE ( "test_eckit_yaml_28" ) { // Value v = YAMLParser::decodeFile("2.28.yaml"); // std::cout << "2.28.yaml " << v << std::endl; // std::cout << toJSON(v) << std::endl; // } // Verify that numeric literals are detected correctly CASE("test_eckit_yaml_29") { EXPECT_EQUAL(YAMLParser::decodeString("1").typeName(), "Number"); EXPECT_EQUAL(YAMLParser::decodeString("12").typeName(), "Number"); EXPECT_EQUAL(YAMLParser::decodeString("1.").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".1").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".12").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1e2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12e3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.e2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.e3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".1e2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".12e3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.2e3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.3e4").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.23e4").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1e-2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12e-3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.e-2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.e-3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".1e-2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".12e-3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.2e-3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.3e-4").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.23e-4").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1e23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12e34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.e23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.e34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".1e23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".12e34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.2e34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.3e45").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.23e45").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1e-23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12e-34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.e-23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.e-34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".1e-23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".12e-34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.2e-34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("12.3e-45").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("1.23e-45").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1").typeName(), "Number"); EXPECT_EQUAL(YAMLParser::decodeString("-12").typeName(), "Number"); EXPECT_EQUAL(YAMLParser::decodeString("-1.").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.1").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.12").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1e2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12e3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.e2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.e3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.1e2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.12e3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.2e3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.3e4").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.23e4").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1e-2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12e-3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.e-2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.e-3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.1e-2").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.12e-3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.2e-3").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.3e-4").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.23e-4").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1e23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12e34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.e23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.e34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.1e23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.12e34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.2e34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.3e45").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.23e45").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1e-23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12e-34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.e-23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.e-34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.1e-23").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-.12e-34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.2e-34").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-12.3e-45").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString("-1.23e-45").typeName(), "Double"); EXPECT_EQUAL(YAMLParser::decodeString(".").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("e").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("e1").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("e12").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("e-1").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("e-12").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString(".e1").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString(".e12").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString(".e-1").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString(".e-12").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-.").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-e").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-e1").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-e12").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-e-1").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-e-12").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-.e1").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-.e12").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-.e-1").typeName(), "String"); EXPECT_EQUAL(YAMLParser::decodeString("-.e-12").typeName(), "String"); } CASE("test_eckit_yaml_cfg_1") { Value v = YAMLParser::decodeFile("cfg.1.yaml"); std::cout << "cfg.1.yaml " << v << std::endl; std::cout << toJSON(v) << std::endl; } #if eckit_HAVE_UNICODE CASE("test_eckit_yaml_unicode") { Value v = YAMLParser::decodeFile("unicode.yaml"); v.dump(std::cout) << std::endl; EXPECT(v["test1"] == v["test2"]); } #endif // eckit_HAVE_UNICODE CASE("test_eckit_yaml_comment_in_string") { Value v = YAMLParser::decodeFile("string.yaml"); std::cout << toJSON(v) << std::endl; EXPECT(v["test1"] == "#fff"); EXPECT(v["test2"] == "#aaa"); std::string e = "starts here and continues here and more"; EXPECT(v["test3"] == e); EXPECT(v["test4"][0] == e); EXPECT(v["test5"]["test6"] == e); EXPECT(v["test6"] == "Quotes in 'strings' are special"); EXPECT(bool(v["bool1"]) == true); EXPECT(bool(v["bool2"]) == false); EXPECT(bool(v["bool3"]) == true); EXPECT(bool(v["bool4"]) == false); EXPECT(bool(v["bool5"]) == true); EXPECT(bool(v["bool6"]) == false); } /// @todo FIX this test // CASE ( "test_eckit_yaml_key_with_space" ) { // Value v = YAMLParser::decodeString("foo : bar"); // EXPECT( v.contains("foo") ); // EXPECT( v["foo"] == Value("bar") ); //} #if __cplusplus > 199711L CASE("test_eckit_ymal_text_1") { const char* text = R"YAML( --- base: &base name: shared1 address: shared2 foo: &foo <<: *base age: 10 bar: &bar <<: *base age: 20 )YAML"; Value v = YAMLParser::decodeString(text); v.dump(std::cout) << std::endl; } CASE("test_eckit_yaml_text_2") { const char* text = R"YAML( --- 1: 2 )YAML"; Value v = YAMLParser::decodeString(text); v.dump(std::cout) << std::endl; ValueMap m(v); EXPECT(v.keys()[0].isNumber()); } CASE("test_eckit_yaml_text_3") { const char* text = R"YAML( --- - '1' - '2' )YAML"; Value v = YAMLParser::decodeString(text); v.dump(std::cout) << std::endl; } CASE("test_eckit_yaml_text_4") { const char* text = R"YAML( --- 165: - 10u - 10 metre u wind component )YAML"; Value v = YAMLParser::decodeString(text); v.dump(std::cout) << std::endl; } CASE("test_eckit_yaml_text_5") { { const char* text = "foo : bar"; Value v = YAMLParser::decodeString(text); v.dump(std::cout) << std::endl; EXPECT(v["foo"] == Value("bar")); } // { // const char* text = "'foo ': bar"; // Value v = YAMLParser::decodeString(text); // v.dump(std::cout) << std::endl; // EXPECT(v["foo "] == Value("bar")); // } // { // const char* text = "'foo ' : bar"; // Value v = YAMLParser::decodeString(text); // v.dump(std::cout) << std::endl; // EXPECT(v["foo "] == Value("bar")); // } } // CASE("test_eckit_yaml_text_6") { // const char* text = R"YAML( // --- // foo: // bar: 1 // spam: true // )YAML"; // EXPECT_THROWS_AS(YAMLParser::decodeString(text), eckit::SeriousBug); // } CASE("test_eckit_yaml_text_7") { const char* text = R"YAML( --- a00: a zero 'a10': a ten 'a11': 'a eleven' "a20": a twenty "a21": "a twenty-one" '30': thirty "40": forty "50": "51": fifty-one 'true': quoted true "false": double quoted false )YAML"; Value v = YAMLParser::decodeString(text); v.dump(std::cout) << std::endl; EXPECT(v.isOrderedMap()); EXPECT(v.keys().size() == 10); EXPECT(v["a00"] == Value("a zero")); EXPECT(v["a10"] == Value("a ten")); EXPECT(v["a11"] == Value("a eleven")); EXPECT(v["a20"] == Value("a twenty")); EXPECT(v["a21"] == Value("a twenty-one")); EXPECT(v["30"] == Value("thirty")); EXPECT(v["40"] == Value("forty")); EXPECT(v["50"]["51"] == Value("fifty-one")); EXPECT(v["true"] == Value("quoted true")); EXPECT(v["false"] == Value("double quoted false")); } #endif //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/parser/2.19.yaml0000664000175000017500000000037515161702250016722 0ustar alastairalastairbase8: 0o32413 base10: 13579 base16: 0x350b base_8: 0o60_150 base_10: 24_680 base_16: 0x6_068 zerobase10: 013579 plus-base10: +13579 minus-base10: -13579 str-base8: "0o32413" str-base10: "13579" str-base16: "0x350b" zero: 0 minus-zero: -0 plus-zero: +0 eckit-2.0.7/tests/parser/2.22.yaml0000664000175000017500000000017315161702250016710 0ustar alastairalastaircanonical: 2001-12-15T02:59:43.1Z iso8601: 2001-12-14t21:59:43.10-05:00 spaced: 2001-12-14 21:59:43.10 -5 date: 2002-12-14 eckit-2.0.7/tests/parser/2.2.yaml0000664000175000017500000000012015161702250016616 0ustar alastairalastairhr: 65 # Home runs avg: 0.278 # Batting average rbi: 147 # Runs Batted In eckit-2.0.7/tests/parser/test_csv.cc0000664000175000017500000000342715161702250017607 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/parser/CSVParser.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_parser_csv_1") { istringstream in( R"CSV(1,2,3 4,5,6)CSV"); CSVParser p(in, false); Value v = p.parse(); std::cout << v << std::endl; } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_parser_csv_2") { istringstream in( R"CSV(a,b,c 1,2,3 4,5,6)CSV"); CSVParser p(in, true); Value v = p.parse(); std::cout << v << std::endl; } //---------------------------------------------------------------------------------------------------------------------- // CASE( "test_eckit_parser_eof" ) { // istringstream in(""); // CSVParser p(in); // EXPECT_THROWS_AS(p.next(), StreamParser::Error); // } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/parser/test_json.cc0000664000175000017500000001032015161702250017753 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit_config.h" #include "eckit/log/JSON.h" #include "eckit/log/Log.h" #include "eckit/parser/JSONParser.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Parse JSON to Value") { std::istringstream in( "{ \"a\" : [true, false, 3], \"b\" : 42.3 , \"c\" : null, \"d\" : \"y\n\tr\rh\", \"e\" : " "\"867017db84f4bc2b5078ca56ffd3b9b9\"}"); JSONParser p(in); Value v = p.parse(); // Log::info() << "json " << v << std::endl; Log::info() << v << std::endl; Log::info() << v["a"] << std::endl; Log::info() << v["a"][2] << std::endl; JSON j(cout); j << v; EXPECT(v.isOrderedMap()); EXPECT(v.as().size() == 5); EXPECT(v["a"].isList()); EXPECT(v["a"].as().size() == 3); EXPECT(v["a"][0].isBool()); EXPECT(v["a"][0].as() == true); EXPECT(v["a"][1].isBool()); EXPECT(v["a"][1].as() == false); EXPECT(v["a"][2].isNumber()); EXPECT((int)v["a"][2] == 3); EXPECT(v["b"].isDouble()); EXPECT(v["b"].as() - 42.3 < 1E-12); EXPECT(v["c"].isNil()); EXPECT(v["d"].isString()); EXPECT(v["e"].isString()); EXPECT(v["e"].as() == "867017db84f4bc2b5078ca56ffd3b9b9"); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_parser_parse_to_set") { istringstream in("[ \"a\" , \"b\", \"c\" ]"); JSONParser p(in); Value v = p.parse(); // Log::info() << "json " << v << std::endl; Log::info() << v << std::endl; EXPECT(v.isList()); EXPECT(v.as().size() == 3); EXPECT(v[0].isString()); EXPECT(v[0].as() == "a"); EXPECT(v[1].isString()); EXPECT(v[1].as() == "b"); EXPECT(v[2].isString()); EXPECT(v[2].as() == "c"); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_parser_parse_to_map") { istringstream in("{ \"a\" : \"AAA\", \"b\" : 0.0 , \"c\" : \"null\", \"d\" : \"\"}"); JSONParser p(in); Value v = p.parse(); // Log::info() << "json " << v << std::endl; Log::info() << v << std::endl; EXPECT(v.isOrderedMap()); EXPECT(v.as().size() == 4); EXPECT(v["a"].isString()); EXPECT(v["a"].as() == "AAA"); EXPECT(v["b"].isDouble()); EXPECT(v["b"].as() == 0.0); EXPECT(v["c"].isString()); EXPECT(v["c"].as() == "null"); EXPECT(v["d"].isString()); EXPECT(v["d"].as() == ""); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_parser_eof") { istringstream in(""); JSONParser p(in); EXPECT_THROWS_AS(p.next(), StreamParser::Error); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_parser_comment_in_string") { istringstream in("{\"test\": \"#fff\"}"); JSONParser p(in); Value v = p.parse(); Log::info() << v << std::endl; EXPECT(v["test"] == "#fff"); } #if eckit_HAVE_UNICODE CASE("test_eckit_parser_unicode") { istringstream in("{\"test\": \"\\u0061\"}"); JSONParser p(in); Value v = p.parse(); Log::info() << v << std::endl; EXPECT(v["test"] == "a"); } #endif // eckit_HAVE_UNICODE //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/parser/2.9.yaml0000664000175000017500000000016315161702250016634 0ustar alastairalastair--- hr: # 1998 hr ranking - Mark McGwire - Sammy Sosa rbi: # 1998 rbi ranking - Sammy Sosa - Ken Griffey eckit-2.0.7/tests/parser/2.28.yaml0000664000175000017500000000063315161702250016717 0ustar alastairalastair--- Time: 2001-11-23 15:01:42 -5 User: ed Warning: This is an error message for the log file --- Time: 2001-11-23 15:02:31 -5 User: ed Warning: A slightly different error message. --- Date: 2001-11-23 15:03:17 -5 User: ed Fatal: Unknown variable "bar" Stack: - file: TopClass.py line: 23 code: | x = MoreObject("345\n") - file: MoreClass.py line: 58 code: |- foo = bar eckit-2.0.7/tests/parser/2.7.yaml0000664000175000017500000000020215161702250016624 0ustar alastairalastair# Ranking of 1998 home runs --- - Mark McGwire - Sammy Sosa - Ken Griffey # Team ranking --- - Chicago Cubs - St Louis Cardinals eckit-2.0.7/tests/parser/2.25.yaml0000664000175000017500000000021515161702250016710 0ustar alastairalastair# sets are represented as a # mapping where each key is # associated with the empty string --- !!set ? Mark McGwire ? Sammy Sosa ? Ken Griff eckit-2.0.7/tests/parser/2.14.yaml0000664000175000017500000000007515161702250016712 0ustar alastairalastair--- Mark McGwire's year was crippled by a knee injury. eckit-2.0.7/tests/parser/cfg.1.yaml0000664000175000017500000000026215161702250017222 0ustar alastairalastair{ "type" : "structured", "yspace" : { "type":"linear", "N":9, "start":90, "end":-90 }, "xspace" : { "type":"linear", "N":16, "start":0, "end":360, "endpoint": false } } eckit-2.0.7/tests/parser/2.26.yaml0000664000175000017500000000023715161702250016715 0ustar alastairalastair# ordered maps are represented as # a sequence of mappings, with # each mapping having one key --- !!omap - Mark McGwire: 65 - Sammy Sosa: 63 - Ken Griffy: 58 eckit-2.0.7/tests/parser/unicode.yaml0000664000175000017500000000003715161702250017752 0ustar alastairalastair--- test1: "\u314A" test2: ㅊ eckit-2.0.7/tests/parser/2.12.yaml0000664000175000017500000000021015161702250016677 0ustar alastairalastair--- # products purchased - item : Super Hoop quantity: 1 - item : Basketball quantity: 4 - item : Big Shoes quantity: 1 eckit-2.0.7/tests/parser/2.3.yaml0000664000175000017500000000020515161702250016623 0ustar alastairalastairamerican: - Boston Red Sox - Detroit Tigers - New York Yankees national: - New York Mets - Chicago Cubs - Atlanta Braves eckit-2.0.7/tests/parser/2.15.yaml0000664000175000017500000000017015161702250016707 0ustar alastairalastair> Sammy Sosa completed another fine season with great stats. 63 Home Runs 0.288 Batting Average What a year! eckit-2.0.7/tests/parser/2.20.yaml0000664000175000017500000000030515161702250016703 0ustar alastairalastaircanonical: 1.23015e+3 exponential: 12.3015e+02 fixed: 1_230.15 infinity: .inf positive infinity: +.inf negative infinity: -.inf not a number: .NaN not a number lower: .nan not a number upper: .NAN eckit-2.0.7/tests/parser/2.23.yaml0000664000175000017500000000041015161702250016703 0ustar alastairalastair--- not-date: !!str 2002-04-28 picture: !!binary | R0lGODlhDAAMAIQAAP//9/X 17unp5WZmZgAAAOfn515eXv Pz7Y6OjuDg4J+fn5OTk6enp 56enmleECcgggoBADs= application specific tag: !something | The semantics of the tag above may be different for different documents. eckit-2.0.7/tests/parser/2.21.yaml0000664000175000017500000000032115161702250016702 0ustar alastairalastairtilde-null: ~ lower-null: null cap-null-: Null upper-null-: NULL lower-true: true cap-true: True upper-true: TRUE y-true: yes lower-false: false cap-false: False upper-false: FALSE n-false: no string: '12345' eckit-2.0.7/tests/parser/2.18.yaml0000664000175000017500000000013615161702250016714 0ustar alastairalastairplain: This unquoted scalar spans many lines. quoted: "So does this quoted scalar.\n" eckit-2.0.7/tests/parser/2.24.yaml0000664000175000017500000000045215161702250016712 0ustar alastairalastair%TAG ! tag:clarkevans.com,2002: --- !shape # Use the ! handle for presenting # tag:clarkevans.com,2002:circle - !circle center: &ORIGIN {x: 73, y: 129} radius: 7 - !line start: *ORIGIN finish: { x: 89, y: 102 } - !label start: *ORIGIN color: 0xFFEEBB text: Pretty vector drawing. eckit-2.0.7/tests/parser/2.27.yaml0000664000175000017500000000120415161702250016711 0ustar alastairalastair--- ! invoice: 34843 date : 2001-01-23 bill-to: &id001 given : Chris family : Dumars address: lines: | 458 Walkman Dr. Suite #292 city : Royal Oak state : MI postal : 48046 ship-to: *id001 product: - sku : BL394D quantity : 4 description : Basketball price : 450.00 - sku : BL4438H quantity : 1 description : Super Hoop price : 2392.00 tax : 251.42 total: 4443.52 comments: Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338. eckit-2.0.7/tests/parser/2.13.yaml0000664000175000017500000000005415161702250016706 0ustar alastairalastair# ASCII Art --- | \//||\/|| // || ||__ eckit-2.0.7/tests/parser/string.yaml0000664000175000017500000000051215161702250017630 0ustar alastairalastair--- test1: '#fff' test2: '#aaa' test3: starts here and continues here and more test4: - starts here and continues here and more test5: test6: starts here and continues here and more test6: 'Quotes in ''strings'' are special' bool1: on bool2: off bool3: true bool4: false bool5: yes bool6: no eckit-2.0.7/tests/parser/CMakeLists.txt0000664000175000017500000000166415161702250020207 0ustar alastairalastairecbuild_add_test( TARGET eckit_test_parser_json SOURCES test_json.cc LIBS eckit ) list( APPEND yaml 2.1.yaml 2.10.yaml 2.11.yaml 2.12.yaml 2.13.yaml 2.14.yaml 2.15.yaml 2.16.yaml 2.17.yaml 2.18.yaml 2.19.yaml 2.2.yaml 2.20.yaml 2.21.yaml 2.22.yaml 2.23.yaml 2.24.yaml 2.25.yaml 2.26.yaml 2.27.yaml 2.28.yaml 2.3.yaml 2.4.yaml 2.5.yaml 2.6.yaml 2.7.yaml 2.8.yaml 2.9.yaml cfg.1.yaml string.yaml unicode.yaml ) foreach( r ${yaml} ) configure_file( ${r} ${CMAKE_CURRENT_BINARY_DIR}/${r} @ONLY ) endforeach() ecbuild_add_test( TARGET eckit_test_parser_yaml SOURCES test_yaml.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_parser_stream_parser SOURCES test_stream_parser.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_parser_csv SOURCES test_csv.cc LIBS eckit ) eckit-2.0.7/tests/parser/2.8.yaml0000664000175000017500000000017515161702250016636 0ustar alastairalastair--- time: 20:03:20 player: Sammy Sosa action: strike (miss) ... --- time: 20:03:47 player: Sammy Sosa action: grand slam ... eckit-2.0.7/tests/types/0000775000175000017500000000000015161702250015310 5ustar alastairalastaireckit-2.0.7/tests/types/test-double-compare-speed.cc0000664000175000017500000000304615161702250022573 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/BigNum.h" #include "eckit/log/Timer.h" #include "eckit/types/FloatCompare.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- void compare(size_t n) { for (size_t i = 0; i < n; ++i) { double x = (double)::rand() / (double)RAND_MAX; eckit::types::is_approximately_equal(x, double(0.5)); } } //---------------------------------------------------------------------------------------------------------------------- CASE("TestDoubleCompareSpeed") { const size_t n = 30000000; // with this data set, on a modern cpu we expect > 25E6 /s eckit::Timer t; compare(n); eckit::Log::info() << "Double compare speed: " << eckit::BigNum(n / t.elapsed()) << " /s" << std::endl; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/types/test_doublecompare.cc0000664000175000017500000002636615161702250021514 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/maths/FloatingPointExceptions.h" #include "eckit/types/FloatCompare.h" #include "eckit/testing/Test.h" namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- namespace { bool is_equal(double a, double b, double epsilon, int maxUlps) { return types::is_approximately_equal(a, b, epsilon, maxUlps); } bool is_equal(double a, double b, double epsilon) { return types::is_approximately_equal(a, b, epsilon); } bool is_equal(double a, double b) { return types::is_approximately_equal(a, b, 0.00001); } const double dEps = std::numeric_limits::epsilon(); const double dInf = std::numeric_limits::infinity(); const double sMin = std::numeric_limits::denorm_min(); const double dMin = std::numeric_limits::min(); const double dMax = std::numeric_limits::max(); const double qNaN = std::numeric_limits::quiet_NaN(); const double sNaN = std::numeric_limits::signaling_NaN(); }; // namespace //---------------------------------------------------------------------------------------------------------------------- CASE("test_large_numbers") { Log::info() << "test_large_numbers" << std::endl; EXPECT(is_equal(1000000, 1000000)); EXPECT(is_equal(1000000, 1000000.00001)); EXPECT(is_equal(1000000.00001, 1000000)); EXPECT(!is_equal(1000000.0, 1000001.0)); EXPECT(!is_equal(1000001.0, 1000000.0)); // ----------------------------------------------- EXPECT(is_equal(dMax, dMax)); EXPECT(is_equal(dMax, dMax, dEps)); EXPECT(is_equal(dMin, dMin)); EXPECT(is_equal(dMin, dMin, dEps)); } CASE("test_negative_large_numbers") { Log::info() << "test_negative_large_numbers " << dMin << std::endl; EXPECT(is_equal(-1000000, -1000000)); EXPECT(is_equal(-1000000, -1000000.00001)); EXPECT(is_equal(-1000000.00001, -1000000)); EXPECT(!is_equal(-1000000.0, -1000001.0)); EXPECT(!is_equal(-1000001.0, -1000000.0)); // ----------------------------------------------- EXPECT(is_equal(-dMax, -dMax)); EXPECT(is_equal(-dMax, -dMax, dEps)); EXPECT(is_equal(-dMin, -dMin)); EXPECT(is_equal(-dMin, -dMin, dEps)); } CASE("test_large_numbers_of_opposite_sign") { EXPECT(!is_equal(-1000000, 1000000)); EXPECT(!is_equal(-1000000, 1000000.00001)); EXPECT(!is_equal(-1000000.00001, 1000000)); EXPECT(!is_equal(-1000000.0, 1000001.0)); EXPECT(!is_equal(-1000001.0, 1000000.0)); // Overflow can occur here (as in CRAY) in eckit::types::is_approximately_equal maths::FloatingPointExceptions::disable_floating_point_exceptions(); EXPECT(!is_equal(-dMax, dMax)); EXPECT(!is_equal(-dMax, dMax, dEps)); maths::FloatingPointExceptions::enable_floating_point_exceptions(); } CASE("test_ulp_around_one") { Log::info() << "test_ulp_around_one" << std::endl; // ULP distances up to 10 are equal // Going right from 1 by eps increases distance by 1 // Going left from 1 by eps increases distance by 2 for (int i = 0; i <= 10; ++i) { EXPECT(is_equal(1.0 + i * dEps, 1.0, dEps)); EXPECT(is_equal(1.0, 1.0 + i * dEps, dEps)); EXPECT(is_equal(1.0 - i * dEps / 2, 1.0, dEps)); EXPECT(is_equal(1.0, 1.0 - i * dEps / 2, dEps)); } // ULP distances greater 10 are not equal EXPECT(!is_equal(1.0 + 11 * dEps, 1.0, dEps)); EXPECT(!is_equal(1.0, 1.0 + 11 * dEps, dEps)); EXPECT(!is_equal(1.0 - 11 * dEps / 2, 1.0, dEps)); EXPECT(!is_equal(1.0, 1.0 - 11 * dEps / 2, dEps)); } CASE("test_numbers_around_one") { Log::info() << "test_numbers_around_one" << std::endl; EXPECT(is_equal(1.0000001, 1.0000002)); EXPECT(is_equal(1.0000002, 1.0000001)); EXPECT(is_equal(1.12345, 1.12346)); EXPECT(is_equal(1.12345, 1.12344, 0.001)); EXPECT(!is_equal(1.0001, 1.0002)); EXPECT(!is_equal(1.0002, 1.0001)); } CASE("test_numbers_around_negative_one") { Log::info() << "test_numbers_around_negative_one" << std::endl; EXPECT(is_equal(-1.0000001, -1.0000002)); EXPECT(is_equal(-1.0000002, -1.0000001)); EXPECT(!is_equal(-1.0001, -1.0002)); EXPECT(!is_equal(-1.0002, -1.0001)); } CASE("test_numbers_between_one_and_zero") { Log::info() << "test_numbers_between_one_and_zero" << std::endl; EXPECT(is_equal(0.000000001000001, 0.000000001000002)); EXPECT(is_equal(0.000000001000002, 0.000000001000001)); EXPECT(!is_equal(0.00102, 0.00101)); EXPECT(!is_equal(0.00101, 0.00102)); } CASE("test_numbers_between_minusone_and_zero") { Log::info() << "test_numbers_between_minusone_and_zero" << std::endl; EXPECT(is_equal(-0.000000001000001, -0.000000001000002)); EXPECT(is_equal(-0.000000001000002, -0.000000001000001)); EXPECT(!is_equal(-0.00102, -0.00101)); EXPECT(!is_equal(-0.00101, -0.00102)); } CASE("test_comparisons_involving_zero") { Log::info() << "test_comparisons_involving_zero" << std::endl; EXPECT(is_equal(0.0, 0.0)); EXPECT(is_equal(0.0, -0.0)); EXPECT(is_equal(-0.0, -0.0)); EXPECT(!is_equal(0.0001, 0.0)); EXPECT(!is_equal(0.0, 0.0001)); EXPECT(!is_equal(-0.0001, 0.0)); EXPECT(!is_equal(0.0, -0.0001)); EXPECT(is_equal(0.0, 1e-40, 0.01)); EXPECT(is_equal(1e-40, 0.0, 0.01)); EXPECT(!is_equal(1e-40, 0.0, 1e-41)); EXPECT(!is_equal(0.0, 1e-40, 1e-41)); EXPECT(is_equal(0.0, -1e-40, 0.1)); EXPECT(is_equal(-1e-40, 0.0, 0.1)); EXPECT(!is_equal(-1e-40, 0.0, 1e-41)); EXPECT(!is_equal(0.0, -1e-40, 1e-41)); } CASE("test_comparisons_involving_infinity") { Log::info() << "test_comparisons_involving_infinity" << std::endl; if (std::numeric_limits::has_infinity) { EXPECT(is_equal(dInf, dInf)); EXPECT(is_equal(-dInf, -dInf)); EXPECT(!is_equal(dInf, dMax)); EXPECT(!is_equal(dMax, dInf)); EXPECT(!is_equal(-dInf, -dMax)); EXPECT(!is_equal(-dMax, -dInf)); } else { Log::info() << "test_comparisons_involving_infinity NOT VALID on this platform" << std::endl; } } CASE("test_comparisons_involving_nan") { Log::info() << "test_comparisons_involving_nan" << std::endl; // The value NaN (Not a Number) is used to represent a value that does not represent a real number. // NaN's are represented by a bit pattern with an exponent of all 1s and a non-zero fraction. T // there are two categories of NaN: QNaN (Quiet NaN) and SNaN (Signalling NaN). // // A QNaN is a NaN with the most significant fraction bit set. // QNaN's propagate freely through most arithmetic operations. // These values pop out of an operation when the result is not mathematically defined. // An SNaN is a NaN with the most significant fraction bit clear. // It is used to signal an exception when used in operations. // SNaN's can be handy to assign to uninitialized variables to trap premature usage. // Semantically, QNaN's denote indeterminate operations, while SNaN's denote invalid operations. EXPECT(!is_equal(qNaN, qNaN)); EXPECT(!is_equal(qNaN, 0.0)); EXPECT(!is_equal(-0.0, qNaN)); EXPECT(!is_equal(qNaN, -0.0)); EXPECT(!is_equal(0.0, qNaN)); EXPECT(!is_equal(qNaN, dInf)); EXPECT(!is_equal(dInf, qNaN)); EXPECT(!is_equal(qNaN, dMax)); EXPECT(!is_equal(dMax, qNaN)); EXPECT(!is_equal(qNaN, -dMax)); EXPECT(!is_equal(-dMax, qNaN)); EXPECT(!is_equal(qNaN, dMin)); EXPECT(!is_equal(dMin, qNaN)); EXPECT(!is_equal(qNaN, -dMin)); EXPECT(!is_equal(-dMin, qNaN)); maths::FloatingPointExceptions::disable_floating_point_exceptions(); EXPECT(!is_equal(sNaN, sNaN)); EXPECT(!is_equal(sNaN, 0.0)); EXPECT(!is_equal(-0.0, sNaN)); EXPECT(!is_equal(sNaN, -0.0)); EXPECT(!is_equal(0.0, sNaN)); EXPECT(!is_equal(sNaN, dInf)); EXPECT(!is_equal(dInf, sNaN)); EXPECT(!is_equal(sNaN, dMax)); EXPECT(!is_equal(dMax, sNaN)); EXPECT(!is_equal(sNaN, -dMax)); EXPECT(!is_equal(-dMax, sNaN)); EXPECT(!is_equal(sNaN, dMin)); EXPECT(!is_equal(dMin, sNaN)); EXPECT(!is_equal(sNaN, -dMin)); EXPECT(!is_equal(-dMin, sNaN)); maths::FloatingPointExceptions::enable_floating_point_exceptions(); } CASE("test_comparisons_opposite_side_of_zero") { Log::info() << "test_comparisons_opposite_side_of_zero" << std::endl; EXPECT(!is_equal(1.000000001, -1.0)); EXPECT(!is_equal(-1.0, 1.000000001)); EXPECT(!is_equal(-1.000000001, 1.0)); EXPECT(!is_equal(1.0, -1.000000001)); EXPECT(is_equal(10.0 * dMin, 10.0 * -dMin)); EXPECT(is_equal(10000 * dMin, 10000 * -dMin)); } CASE("test_comparisons_very_close_to_zero") { Log::info() << "test_comparisons_very_close_to_zero" << std::endl; EXPECT(is_equal(dMin, -dMin, dEps)); EXPECT(is_equal(-dMin, dMin, dEps)); EXPECT(is_equal(dMin, 0, dEps)); EXPECT(is_equal(0, dMin, dEps)); EXPECT(is_equal(-dMin, 0, dEps)); EXPECT(is_equal(0, -dMin, dEps)); EXPECT(is_equal(0.000000001, -dMin)); EXPECT(is_equal(0.000000001, dMin)); EXPECT(is_equal(dMin, 0.000000001)); EXPECT(is_equal(-dMin, 0.000000001)); EXPECT(!is_equal(0.000000001, -dMin, 1e-10)); EXPECT(!is_equal(0.000000001, dMin, 1e-10)); EXPECT(!is_equal(dMin, 0.000000001, 1e-10)); EXPECT(!is_equal(-dMin, 0.000000001, 1e-10)); } CASE("test_comparisons_with_denormal_numbers") { Log::info() << "test_comparisons_with_denormal_numbers" << std::endl; EXPECT(is_equal(sMin, -sMin, dEps)); EXPECT(is_equal(-sMin, sMin, dEps)); EXPECT(is_equal(sMin, 0, dEps)); EXPECT(is_equal(0, sMin, dEps)); EXPECT(is_equal(-sMin, 0, dEps)); EXPECT(is_equal(0, -sMin, dEps)); const double lMin = dMin - sMin; // largest denormal number EXPECT(is_equal(lMin, -lMin, dEps)); EXPECT(is_equal(-lMin, lMin, dEps)); EXPECT(is_equal(lMin, 0, dEps)); EXPECT(is_equal(0, lMin, dEps)); EXPECT(is_equal(-lMin, 0, dEps)); EXPECT(is_equal(0, -lMin, dEps)); } CASE("test_comparisons_ulps") { Log::info() << "test_comparisons_ulps" << std::endl; EXPECT(is_equal(dMin, -dMin, 0, 2)); EXPECT(is_equal(-dMin, dMin, 0, 2)); EXPECT(is_equal(dMin, 0, 0, 1)); EXPECT(is_equal(0, dMin, 0, 1)); EXPECT(is_equal(-dMin, 0, 0, 1)); EXPECT(is_equal(0, -dMin, 0, 1)); EXPECT(!is_equal(dMin, -dMin, 0, 1)); EXPECT(!is_equal(-dMin, dMin, 0, 1)); EXPECT(!is_equal(dMin, 0, 0, 0)); EXPECT(!is_equal(0, dMin, 0, 0)); EXPECT(!is_equal(-dMin, 0, 0, 0)); EXPECT(!is_equal(0, -dMin, 0, 0)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { eckit::maths::FloatingPointExceptions::enable_floating_point_exceptions(); return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/types/test_fixedstring.cc0000664000175000017500000003210715161702250021207 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/types/FixedString.h" #include "eckit/testing/Test.h" #ifdef __clang__ #pragma clang diagnostic ignored "-Wself-assign-overloaded" #endif using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_constructor_zero") { // What happens if we create a length-zero FixedString? FixedString<0> fs; EXPECT(fs.length() == 0); EXPECT(fs.size() == 0); EXPECT(fs.asString() == ""); EXPECT(std::string(fs) == ""); FixedString<0> fs2(""); EXPECT(fs2.length() == 0); EXPECT(fs2.size() == 0); EXPECT(fs2.asString() == ""); EXPECT(std::string(fs2) == ""); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_constructor_default") { FixedString<8> fs; EXPECT(fs.length() == 0); EXPECT(fs.asString() == ""); EXPECT(std::string(fs) == ""); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_constructor_string") { // Construct with a string of the correct length std::string str1("12345678"); FixedString<8> fs1(str1); EXPECT(fs1.size() == 8); EXPECT(fs1.length() == 8); EXPECT(fs1.asString() == "12345678"); EXPECT(std::string(fs1) == "12345678"); // Construct with a string that is too short std::string str2("1234"); FixedString<8> fs2(str2); EXPECT(fs2.size() == 8); EXPECT(fs2.length() == 4); EXPECT(fs2.asString() == "1234"); EXPECT(std::string(fs2) == "1234"); // Construct with a string that is too long // n.b. use functor to make exception thrown in constructor palatable to EXPECT_ THR OW struct fs3_allocate { void operator()() { std::string str3("1234567890"); FixedString<8> fs3(str3); } }; EXPECT_THROWS_AS(fs3_allocate()(), AssertionFailed); // Construct with a zero length string std::string str4; FixedString<8> fs4(str4); EXPECT(fs4.size() == 8); EXPECT(fs4.length() == 0); EXPECT(fs4.asString() == ""); EXPECT(std::string(fs4) == ""); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_constructor_char") { // Construct with a string of the correct length FixedString<8> fs1("12345678"); EXPECT(fs1.size() == 8); EXPECT(fs1.length() == 8); EXPECT(fs1.asString() == "12345678"); EXPECT(std::string(fs1) == "12345678"); // Construct with a string that is too short FixedString<8> fs2("1234"); EXPECT(fs2.size() == 8); EXPECT(fs2.length() == 4); EXPECT(fs2.asString() == "1234"); EXPECT(std::string(fs2) == "1234"); // Construct with a string that is too long // n.b. use functor to make exception thrown in constructor palatable to EXPECT_ THR OW struct fs3_allocate { void operator()() { FixedString<8> fs3("1234567890"); } }; EXPECT_THROWS_AS(fs3_allocate()(), AssertionFailed); // Construct with a zero length string FixedString<8> fs4(""); EXPECT(fs4.size() == 8); EXPECT(fs4.length() == 0); EXPECT(fs4.asString() == ""); EXPECT(std::string(fs4) == ""); // Construct with a null pointer struct fs5_allocate { void operator()() { FixedString<8> fs5(nullptr); } }; EXPECT_THROWS_AS(fs5_allocate()(), AssertionFailed); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_constructor_fixedstring") { // Construct with a string of the correct length /// @note for all of these assignments, except for the <8> --> <8> the std::string constructor is used. It might /// be better for the long term if a template FixedString(const FixedString& ) /// constructor existed. std::string str1("12345678"); FixedString<8> fs1(str1); FixedString<8> fs1a(fs1); EXPECT(fs1a.size() == 8); EXPECT(fs1a.length() == 8); EXPECT(fs1a.asString() == "12345678"); EXPECT(std::string(fs1a) == "12345678"); // Construct with a string that is too short std::string str2("1234"); FixedString<4> fs2(str2); FixedString<8> fs2a(fs2); EXPECT(fs2a.size() == 8); EXPECT(fs2a.length() == 4); EXPECT(fs2a.asString() == "1234"); EXPECT(std::string(fs2a) == "1234"); // Construct with a string that is too long // n.b. use functor to make exception thrown in constructor palatable to EXPECT_ THR OW struct fs3_allocate { void operator()() { std::string str3("1234567890"); FixedString<10> fs3(str3); FixedString<8> fs3a(fs3); } }; EXPECT_THROWS_AS(fs3_allocate()(), AssertionFailed); // Construct with a zero length string std::string str4; FixedString<0> fs4(str4); FixedString<8> fs4a(fs4); EXPECT(fs4a.size() == 8); EXPECT(fs4a.length() == 0); EXPECT(fs4a.asString() == ""); EXPECT(std::string(fs4a) == ""); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_assignment_string") { // Construct with a string of the correct length FixedString<8> fs1 = std::string("12345678"); EXPECT(fs1.size() == 8); EXPECT(fs1.length() == 8); EXPECT(fs1.asString() == "12345678"); EXPECT(std::string(fs1) == "12345678"); // Construct with a string that is too short FixedString<8> fs2 = std::string("1234"); EXPECT(fs2.size() == 8); EXPECT(fs2.length() == 4); EXPECT(fs2.asString() == "1234"); EXPECT(std::string(fs2) == "1234"); // Construct with a string that is too long // n.b. use functor to make exception thrown in assignment palatable to EXPECT_ THR OW struct fs3_assign { void operator()() { FixedString<8> fs3 = std::string("1234567890"); (void)fs3; } }; EXPECT_THROWS_AS(fs3_assign()(), AssertionFailed); // Construct with a zero length string FixedString<8> fs4 = std::string(); EXPECT(fs4.size() == 8); EXPECT(fs4.length() == 0); EXPECT(fs4.asString() == ""); EXPECT(std::string(fs4) == ""); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_assignment_fixedstring") { // Construct with a string of the correct length /// @note for all of these assignments, except for the <8> --> <8> the std::string constructor is used. It might /// be better for the long term if a template FixedString(const FixedString& ) /// constructor existed. FixedString<8> fs1("12345678"); FixedString<8> fs1a = fs1; EXPECT(fs1a.size() == 8); EXPECT(fs1a.length() == 8); EXPECT(fs1a.asString() == "12345678"); EXPECT(std::string(fs1a) == "12345678"); /// Mismatched size assignments are (correctly) compile-time disallowed. /// // Construct with a string that is too short /// FixedString<4> fs2("1234"); /// FixedString<8> fs2a = fs2; /// EXPECT( fs2a.size() == 8 ); /// EXPECT( fs2a.length() == 4 ); /// EXPECT( fs2a.asString() == "1234" ); /// EXPECT( std::string(fs2a) == "1234" ); /// // Construct with a string that is too long /// // n.b. use functor to make exception thrown in constructor palatable to EXPECT_ THR OW /// struct fs3_allocate { /// void operator()() { /// FixedString<10> fs3("1234567890"); /// FixedString<8> fs3a = fs3; /// } /// }; /// EXPECT_THROWS_AS( fs3_allocate()(), AssertionFailed ); /// // Construct with a zero length string /// FixedString<0> fs4; /// FixedString<8> fs4a = fs4; /// EXPECT( fs4a.size() == 8 ); /// EXPECT( fs4a.length() == 0 ); /// EXPECT( fs4a.asString() == "" ); /// EXPECT( std::string(fs4a) == "" ); // self-assignment is ok fs1a = fs1a; EXPECT(fs1a.size() == 8); EXPECT(fs1a.length() == 8); EXPECT(fs1a.asString() == "12345678"); EXPECT(std::string(fs1a) == "12345678"); } //---------------------------------------------------------------------------------------------------------------------- // // Test comparison operators // //---------------------------------------------------------------------------------------------------------------------- /// @note Comparisons between different FixedString, FixedString not permitted at compile time. CASE("test_eckit_types_fixedstring_operators") { FixedString<8> fs1("12345678"); FixedString<8> fs2("12345678"); FixedString<8> fs3("abcdefgh"); // == EXPECT(fs1 == fs1); EXPECT(fs1 == fs2); EXPECT(!(fs1 == fs3)); // != EXPECT(!(fs1 != fs1)); EXPECT(!(fs1 != fs2)); EXPECT(fs1 != fs3); // < EXPECT(!(fs1 < fs1)); EXPECT(!(fs1 < fs2)); EXPECT(fs1 < fs3); EXPECT(!(fs3 < fs1)); // > EXPECT(!(fs1 > fs1)); EXPECT(!(fs1 > fs2)); EXPECT(!(fs1 > fs3)); EXPECT(fs3 > fs1); // <= EXPECT(fs1 <= fs1); EXPECT(fs1 <= fs2); EXPECT(fs1 <= fs3); EXPECT(!(fs3 <= fs1)); // >= EXPECT(fs1 >= fs1); EXPECT(fs1 >= fs2); EXPECT(!(fs1 >= fs3)); EXPECT(fs3 >= fs1); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_operators_zero") { FixedString<0> fs1; FixedString<0> fs2; // == EXPECT(fs1 == fs1); EXPECT(fs1 == fs2); // != EXPECT(!(fs1 != fs1)); EXPECT(!(fs1 != fs2)); // < EXPECT(!(fs1 < fs1)); EXPECT(!(fs1 < fs2)); // > EXPECT(!(fs1 > fs1)); EXPECT(!(fs1 > fs2)); // <= EXPECT(fs1 <= fs1); EXPECT(fs1 <= fs2); // >= EXPECT(fs1 >= fs1); EXPECT(fs1 >= fs2); } //---------------------------------------------------------------------------------------------------------------------- // // Test extraneous bits // //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_byte_size") { FixedString<0> fs0; FixedString<1> fs1; FixedString<4> fs4; FixedString<8> fs8; EXPECT(sizeof(fs0) == 0); EXPECT(sizeof(fs1) == 1); EXPECT(sizeof(fs4) == 4); EXPECT(sizeof(fs8) == 8); EXPECT(sizeof(FixedString<0>) == 0); EXPECT(sizeof(FixedString<1>) == 1); EXPECT(sizeof(FixedString<4>) == 4); EXPECT(sizeof(FixedString<8>) == 8); EXPECT(fs0.size() == 0); EXPECT(fs1.size() == 1); EXPECT(fs4.size() == 4); EXPECT(fs8.size() == 8); EXPECT(FixedString<0>::static_size() == 0); EXPECT(FixedString<1>::static_size() == 1); EXPECT(FixedString<4>::static_size() == 4); EXPECT(FixedString<8>::static_size() == 8); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_data_access") { FixedString<8> fs("12345678"); // Check that const and non-const pointers give access to same data... char* d = fs.data(); const char* cd = fs.data(); // EXPECT does a string comparison for cha r* EXPECT(d == cd); // Check that if we insert a \0 character appropriately, the accessible strings and lengths adjust correctly EXPECT(std::string(fs) == "12345678"); EXPECT(fs.length() == 8); d[4] = '\0'; EXPECT(std::string(fs) == "1234"); EXPECT(fs.length() == 4); d[6] = '\0'; d[4] = 'F'; EXPECT(std::string(fs) == "1234F6"); EXPECT(fs.length() == 6); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_fixedstring_reassign_shorter_string") { ///< @see ECKIT-182 FixedString<64> fs; fs = std::string("calvin"); EXPECT(fs == std::string("calvin")); /* this worked */ fs = std::string("calvin & hobbes"); EXPECT(fs == std::string("calvin & hobbes")); /* assinging longer string also worked */ fs = std::string("susie"); EXPECT(fs == std::string("susie")); /* but then assigning shorter did not, resulted "susien & hobbes" */ } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/types/test_print_vector.cc0000664000175000017500000000510115161702250021371 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/types/Types.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_print_vector_string") { vector vstr; vstr.push_back("Hello"); vstr.push_back("World"); vstr.push_back("test"); vstr.push_back("case"); stringstream s; s << vstr; EXPECT("[Hello,World,test,case]" == s.str()); } CASE("test_eckit_print_vector_pair") { vector > vpair; vpair.push_back(make_pair("k1", 123)); vpair.push_back(make_pair("k1", 124)); vpair.push_back(make_pair("k1", 125)); vpair.push_back(make_pair("k2", 125)); stringstream s; s << vpair; EXPECT("[,,,]" == s.str()); } CASE("test_eckit_print_vector_ints") { vector vint; vint.push_back(123); vint.push_back(124); vint.push_back(125); vint.push_back(126); vint.push_back(127); vint.push_back(129); vint.push_back(131); vint.push_back(133); vint.push_back(135); vint.push_back(135); vint.push_back(135); vint.push_back(135); vint.push_back(1); stringstream s; s << vint; EXPECT("[123-127,129-135-2,3*135,1]" == s.str()); } CASE("test_eckit_print_vector_doubs") { // These should not contract into ranges. vector vdoub; vdoub.push_back(123.0); vdoub.push_back(124.0); vdoub.push_back(125.0); vdoub.push_back(126.0); vdoub.push_back(127.0); vdoub.push_back(129.0); vdoub.push_back(131.0); vdoub.push_back(133.0); vdoub.push_back(135.0); vdoub.push_back(135.0); vdoub.push_back(135.0); vdoub.push_back(135.0); vdoub.push_back(1.0); stringstream s; s << vdoub; EXPECT("[123,124,125,126,127,129,131,133,135,135,135,135,1]" == s.str()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/types/test_hour.cc0000664000175000017500000000573115161702250017641 0ustar alastairalastair/* * (C) Copyright 2021- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" #include "eckit/types/Hour.h" using namespace std; using namespace eckit; using namespace eckit::testing; using eckit::types::is_approximately_equal; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Basic test") { EXPECT(Hour(12) == Hour("12")); EXPECT(Hour(12.5) == Hour("12:30")); EXPECT(Hour(1.0 / 3.0) == Hour("0:20")); EXPECT(Hour(2.0 / 3.0) == Hour("0:40")); EXPECT(Hour(1.0 / 60.0) == Hour("0:01")); EXPECT(Hour(1.0 / 3600.0) == Hour("0:00:01")); } CASE("Test minutes") { for (int hour = 0; hour < 240; ++hour) { for (int minute = 0; minute < 60; ++minute) { std::ostringstream oss; if (minute == 0) { oss << hour; } else { oss << hour << ':' << std::setw(2) << std::setfill('0') << minute; } std::string s = oss.str(); // std::cout << s << std::endl; // std::cout << Hour(s) << std::endl; EXPECT(Hour(s).asString() == s); double d = double(hour) + double(minute) / 60.0; EXPECT(Hour(d).asString() == s); } } } CASE("Test seconds") { for (int hour = 0; hour < 24; ++hour) { for (int minute = 0; minute < 60; ++minute) { for (int second = 0; second < 60; ++second) { std::ostringstream oss; if (second == 0) { if (minute == 0) { oss << hour; } else { oss << hour << ':' << std::setw(2) << std::setfill('0') << minute; } } else { oss << hour << ':' << std::setw(2) << std::setfill('0') << minute << ':' << std::setw(2) << std::setfill('0') << second; } std::string s = oss.str(); // std::cout << s << std::endl; // std::cout << Hour(s) << std::endl; EXPECT(Hour(s).asString() == s); double d = double(hour) + double(minute) / 60.0 + double(second) / 3600.0; EXPECT(Hour(d).asString() == s); } } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/types/test_uuid.cc0000664000175000017500000000415215161702250017626 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/types/UUID.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_uuid_default_constructor") { UUID uuid; EXPECT(uuid.size() == 16); EXPECT(uuid.isNil()); std::string res("00000000000000000000000000000000"); EXPECT(res == uuid.asString()); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_uuid_constructor_string") { std::string s("4b4053dc93e0b52a6f028cb36649d229"); UUID uuid(s); EXPECT(uuid.size() == 16); EXPECT(!uuid.isNil()); EXPECT(s == uuid.asString()); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_uuid_fromstring") { std::string s("4b4053dc93e0b52a6f028cb36649d229"); UUID uuid; EXPECT(uuid.isNil()); EXPECT_NO_THROW(uuid.fromString(s)); EXPECT(!uuid.isNil()); EXPECT(s == uuid.asString()); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_types_uuid_constructor_string_trow") { std::string s("4b405"); EXPECT_THROWS_AS(UUID uuid(s), eckit::AssertionFailed); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/types/test_time.cc0000664000175000017500000001372415161702250017623 0ustar alastairalastair/* * (C) Copyright 2021- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/testing/Test.h" #include "eckit/types/Time.h" using namespace std; using namespace eckit; using namespace eckit::testing; using eckit::types::is_approximately_equal; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Time only digits (hhmmss)") { EXPECT(Time(0, 0, 0) == Time("0")); EXPECT(Time(0, 0, 0) == Time("00")); EXPECT(Time(0, 0, 0) == Time("000")); EXPECT(Time(0, 0, 0) == Time("0000")); EXPECT(Time(0, 0, 0) == Time("00000")); EXPECT(Time(0, 0, 0) == Time("000000")); EXPECT(Time(0, 0, 0) == Time(0)); EXPECT(Time(0, 0, 1) == Time(1)); EXPECT(Time(0, 1, 0) == Time(60)); EXPECT(Time(0, 1, 1) == Time(61)); EXPECT(Time(1, 0, 0) == Time(3600)); EXPECT(Time(1, 0, 1) == Time(3601)); EXPECT(Time(1, 1, 0) == Time(3660)); EXPECT(Time(1, 1, 1) == Time(3661)); EXPECT(Time(2, 3, 4) == Time(3600 * 2 + 60 * 3 + 4)); EXPECT_THROWS(Time(24 * 3600)); EXPECT_THROWS(Time(24, 0, 0)); EXPECT_THROWS(Time("24")); EXPECT_THROWS(Time(-1)); EXPECT_THROWS(Time("-1")); EXPECT_NO_THROW(Time(-1, 0, 0, true)); EXPECT_NO_THROW(Time("-1", true)); EXPECT(Time(-1, 0, 0, true) == Time("-1", true)); EXPECT(Time(24, 0, 0, true) == Time(24 * 3600, true)); EXPECT(Time(24, 0, 0, true) == Time("24", true)); EXPECT(Time(-1, 0, 0, true) == Time("-1", true)); EXPECT(Time(-1, 0, 0, true) == Time("-01", true)); EXPECT(Time(-100, 0, 0, true) == Time("-100", true)); EXPECT(Time(-100, 0, 0, true) == Time("-0100", true)); EXPECT(Time(0, -30, 0, true) == Time("-0.5", true)); EXPECT(Time(2, 0, 0) == Time("2")); EXPECT(Time(2, 0, 0) == Time("02")); EXPECT(Time(2, 0, 0) == Time("200")); EXPECT(Time(2, 0, 0) == Time("0200")); EXPECT(Time(2, 0, 0) == Time("20000")); EXPECT(Time(2, 0, 0) == Time("020000")); EXPECT(Time(20, 0, 0) == Time("20")); EXPECT(Time(20, 0, 0) == Time("2000")); EXPECT(Time(20, 0, 0) == Time("200000")); EXPECT(Time(20, 0, 0) == Time("20", true)); EXPECT_THROWS(Time(30, 0, 0)); EXPECT_THROWS(Time("30")); EXPECT_THROWS(Time("3000")); EXPECT_THROWS(Time("300000")); EXPECT(Time(30, 0, 0, true) == Time("30", true)); EXPECT(Time(0, 3, 0) == Time("003")); EXPECT(Time(0, 3, 0) == Time("0003")); EXPECT(Time(0, 3, 0) == Time("00300")); EXPECT(Time(0, 3, 0) == Time("000300")); EXPECT(Time(0, 0, 4) == Time("00004")); EXPECT(Time(0, 0, 4) == Time("000004")); EXPECT(Time(1, 23, 0) == Time("123")); EXPECT(Time(1, 23, 0) == Time("0123")); EXPECT(Time(1, 23, 0) == Time("12300")); EXPECT(Time(1, 23, 0) == Time("012300")); EXPECT(Time(1, 23, 45) == Time("12345")); EXPECT(Time(1, 23, 45) == Time("012345")); EXPECT_THROWS(Time("25")); EXPECT_THROWS(Time("175")); EXPECT_THROWS(Time("0175")); EXPECT_THROWS(Time("3025")); EXPECT_THROWS(Time("017345")); EXPECT_THROWS(Time("012375")); } CASE("Time format (hh:mm:ss)") { EXPECT(Time(0, 0, 0) == Time("0:0")); EXPECT(Time(0, 0, 0) == Time("0:00")); EXPECT(Time(0, 0, 0) == Time("00:0")); EXPECT(Time(0, 0, 0) == Time("00:00")); EXPECT(Time(0, 0, 0) == Time("0:00:00")); EXPECT(Time(0, 0, 0) == Time("00:0:00")); EXPECT(Time(0, 0, 0) == Time("00:00:0")); EXPECT(Time(0, 0, 0) == Time("00:00:00")); EXPECT(Time(2, 0, 0) == Time("2:0")); EXPECT(Time(2, 0, 0) == Time("02:0")); EXPECT(Time(2, 0, 0) == Time("2:00")); EXPECT(Time(2, 0, 0) == Time("02:00")); EXPECT(Time(2, 0, 0) == Time("2:00:00")); EXPECT(Time(2, 0, 0) == Time("02:00:00")); EXPECT(Time(0, 3, 0) == Time("00:3")); EXPECT(Time(0, 3, 0) == Time("00:03")); EXPECT(Time(0, 3, 0) == Time("0:03:00")); EXPECT(Time(0, 3, 0) == Time("00:3:00")); EXPECT(Time(0, 3, 0) == Time("00:03:00")); EXPECT(Time(0, 0, 4) == Time("00:00:4")); EXPECT(Time(0, 0, 4) == Time("00:00:04")); EXPECT(Time(1, 23, 0) == Time("1:23")); EXPECT(Time(1, 23, 0) == Time("01:23")); EXPECT(Time(1, 23, 0) == Time("1:23:00")); EXPECT(Time(1, 23, 0) == Time("01:23:00")); EXPECT(Time(1, 23, 45) == Time("1:23:45")); EXPECT(Time(1, 23, 45) == Time("01:23:45")); EXPECT_THROWS(Time("25")); EXPECT_THROWS(Time("175")); EXPECT_THROWS(Time("0175")); EXPECT_THROWS(Time("3025")); EXPECT_THROWS(Time("017345")); EXPECT_THROWS(Time("012375")); } CASE("Time with unit (__h__m__s)") { EXPECT(Time(2, 0, 0) == Time("2h")); EXPECT(Time(2, 0, 0) == Time("0002H")); EXPECT(Time(2, 0, 0) == Time("120m")); EXPECT(Time(2, 0, 0) == Time("7200s")); EXPECT(Time(0, 3, 0) == Time("3M")); EXPECT(Time(0, 3, 0) == Time("180s")); EXPECT(Time(1, 2, 3) == Time("1h2m3s")); EXPECT(Time(1, 23, 45) == Time("1h23m45s")); EXPECT(Time(1, 23, 45) == Time("01h23m45s")); EXPECT(Time(1, 23, 45) == Time("5025s")); EXPECT(Time(1, 23, 45) == Time("83m45s")); EXPECT_THROWS(Time("25h")); EXPECT_NO_THROW(Time("25h", true)); EXPECT(Time(0, -30, 0, true) == Time("-30m", true)); EXPECT(Time(-1, -30, 0, true) == Time("-1h30m", true)); EXPECT(Time(0, -90, 0, true) == Time("-1h30m", true)); EXPECT(Time("0d3h10m20s") == Time("3h10m20s")); EXPECT(Time("0d3h10m20s") == Time("3h620s")); EXPECT(Time("2D3h", true) == Time("51h", true)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/types/test_fraction.cc0000664000175000017500000002274315161702250020473 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/types/Fraction.h" #include "eckit/utils/Translator.h" #include "eckit/testing/Test.h" namespace eckit::test { auto& LOG = Log::info(); //---------------------------------------------------------------------------------------------------------------------- CASE("Constructing fractions") { // 0 EXPECT(Fraction(0, 1) == Fraction()); EXPECT_THROWS_AS(Fraction(0, 0), std::exception); // this prints a backtrace wi Assertion failed // negative number EXPECT(Fraction(-1, 2) == Fraction(3, -6)); // decimals EXPECT(Fraction(0.16) == Fraction(16, 100)); EXPECT(Fraction(0.1616) == Fraction(1616, 10000)); // 5 / 7 EXPECT(Fraction(0.714285) != Fraction(5, 7)); EXPECT(Fraction(0.7142857142) != Fraction(5, 7)); EXPECT(Fraction(0.71428571428) == Fraction(5, 7)); EXPECT(Fraction(0.714285714285) == Fraction(5, 7)); EXPECT(Fraction(0.714285714286) == Fraction(5, 7)); EXPECT(Fraction(0.714285714285714285) == Fraction(5, 7)); // 1 / 6 EXPECT(Fraction(0.166) != Fraction(1, 6)); EXPECT(Fraction(0.1666) != Fraction(1, 6)); EXPECT(Fraction(0.16666) != Fraction(1, 6)); EXPECT(Fraction(0.166666) != Fraction(1, 6)); EXPECT(Fraction(0.1666666) != Fraction(1, 6)); EXPECT(Fraction(0.16666666) != Fraction(1, 6)); EXPECT(Fraction(0.166666666) != Fraction(1, 6)); EXPECT(Fraction(0.16666666666) == Fraction(1, 6)); EXPECT(Fraction(0.166666666666) == Fraction(1, 6)); EXPECT(Fraction(0.1666666666666) == Fraction(1, 6)); EXPECT(Fraction(0.16666666666666) == Fraction(1, 6)); EXPECT(Fraction(0.166666666666666) == Fraction(1, 6)); EXPECT(Fraction(0.1666666666666666) == Fraction(1, 6)); // 1 / 3 EXPECT(Fraction(0.3333333333) == Fraction(1, 3)); EXPECT(Fraction(0.3333333333333333) == Fraction(1, 3)); // 1 / 1 EXPECT(Fraction(0.9999999999999999) == Fraction(1, 1)); EXPECT(Fraction(0.9999999999999999) == Fraction(10, 10)); // 7 / 10.. EXPECT(Fraction(0.7) == Fraction(7, 10)); EXPECT(Fraction(0.07) == Fraction(7, 100)); EXPECT(Fraction(0.0000007) == Fraction(7, 10000000)); // operations EXPECT(Fraction(1, 3) + Fraction(2, 3) == Fraction(10, 10)); EXPECT(Fraction(1, 3) - Fraction(2, 6) == Fraction(0, 10)); EXPECT(Fraction(1, 3) * 3 == Fraction(1)); EXPECT(-Fraction(1, 3) == Fraction(1, -3)); EXPECT(2 * Fraction(1, 3) == Fraction(2, 3)); Fraction a(1, 3); Fraction b(3, 28); EXPECT(a + b == Fraction(37, 84)); EXPECT(a - b == Fraction(19, 84)); EXPECT(a * b == Fraction(1, 28)); EXPECT(a / b == Fraction(28, 9)); EXPECT(a > b); EXPECT(a != b); EXPECT_NOT(a < b); EXPECT_NOT(a == b); EXPECT(Fraction("1/3") == Fraction(1, 3)); EXPECT(double(Fraction("1/3")) == 1.0 / 3.0); EXPECT(Fraction("1") == Fraction(1)); EXPECT(Fraction("1.2") == Fraction(12, 10)); EXPECT(Fraction("1e-6") == Fraction(1, 1000000)); EXPECT(Fraction("1e+6") == Fraction(1000000, 1)); EXPECT(Fraction("1.2e+6") == Fraction(1200000, 1)); EXPECT(Fraction(M_PI) > Fraction(M_E)); EXPECT(Fraction(M_E) > Fraction(M_SQRT2)); LOG << "pi = " << (M_PI - double(Fraction(M_PI))) << std::endl; LOG << "e = " << (M_E - double(Fraction(M_E))) << std::endl; LOG << "sqrt2 = " << (M_SQRT2 - double(Fraction(M_SQRT2))) << std::endl; // EXPECT(Fraction(M_PI), Fraction(1200000, 1)); { Fraction west(-12); Fraction east(1.2); Fraction increment(1.2); Fraction f(west); while (f < east) { f += increment; } EXPECT(f == east); } { Fraction west("-77"); Fraction east("7"); Fraction increment("0.7"); Fraction f(west); while (f < east) { f += increment; } EXPECT(f == east); } EXPECT(Fraction(241.85) / Fraction(0.175) == 1382); EXPECT(Fraction(284.025) / Fraction(0.175) == 1623); // EXPECT(Fraction(5, 3).intergralPart() == 1); // EXPECT(Fraction(5, 3).decimalPart() == Fraction(1, 3)); } //---------------------------------------------------------------------------------------------------------------------- CASE("Overflow during comparisons") { { Fraction A(5934563467713522511, 13567822205000000); // 437.39985519021053 Fraction B(8624662771, 19718023); // 437.3999751902105 EXPECT(A < B); EXPECT(A <= B); EXPECT(A != B); EXPECT(B > A); EXPECT(B >= A); } { constexpr auto MAX = std::numeric_limits::max(); Fraction A(MAX - 6, MAX - 1); Fraction B(MAX - 2, MAX - 1); LOG << MAX - 6 << " ?= " << static_cast(MAX - 6) << std::endl; LOG << MAX - 2 << " ?= " << static_cast(MAX - 2) << std::endl; /// @note this demonstrates that numerators/denominators have a lossy representation in double EXPECT(double(MAX - 6) == double(MAX - 2)); EXPECT(A != B); /// @note these fail due to the lossy conversion to double of numerators/denominators // EXPECT(A < B); // EXPECT(A <= B); // EXPECT(B > A); // EXPECT(B >= A); LOG << "Max denominator" << Fraction::max_denominator() << std::endl; Fraction U(1, 1); /// @note these fail due to the lossy conversion to double of numerators/denominators EXPECT(A != U); EXPECT(A < U); EXPECT(A <= U); EXPECT(U > A); EXPECT(U >= A); EXPECT(B != U); EXPECT(B < U); EXPECT(B <= U); EXPECT(U > B); EXPECT(U >= B); } } //---------------------------------------------------------------------------------------------------------------------- CASE("Regression (Fraction <=> Fraction)") { Fraction A(-34093871309, 378680550); // -90.033331 Fraction B(-12621809, 378680550); // -0.3333102 EXPECT(A < B); EXPECT(A <= B); EXPECT(A != B); EXPECT(B > A); EXPECT(B >= A); } //---------------------------------------------------------------------------------------------------------------------- CASE("Regression (double <=> Fraction)") { double A(253.07432399999999006467987783253); Fraction B(/*253.07435716708162*/ 1591682572, 6289387); EXPECT(A < B); EXPECT(A <= B); EXPECT(A != B); EXPECT(B > A); EXPECT(B >= A); } //---------------------------------------------------------------------------------------------------------------------- CASE("Regression (Fraction <=> double)") { Fraction A(/*-253.07435716708162*/ -1591682572, 6289387); double B(-253.07432399999999006467987783253); EXPECT(A < B); EXPECT(A <= B); EXPECT(A != B); EXPECT(B > A); EXPECT(B >= A); } //---------------------------------------------------------------------------------------------------------------------- CASE("Values known to have problematic conversion to fraction") { auto values = std::vector{19.011363983154297, 0.47718059708975263}; for (auto value : values) { LOG << "Test " << value << "..." << std::endl; auto frac = Fraction(value); LOG << "Test " << value << " = " << frac << std::endl; EXPECT_NOT_EQUAL(frac.denominator(), 0); } } //---------------------------------------------------------------------------------------------------------------------- CASE("Fraction safe conversion") { constexpr auto safe = static_cast(std::numeric_limits::max()); EXPECT_NO_THROW(Fraction{safe}); constexpr auto unsafe = safe + 1; EXPECT_THROWS_AS(Fraction{unsafe}, BadCast); } //---------------------------------------------------------------------------------------------------------------------- CASE("Fraction inverse") { for (const auto& test : { std::make_pair(Fraction{1, 2}, Fraction{2, 1}), std::make_pair(Fraction{1, 3}, Fraction{3, 1}), }) { LOG << "Test (" << test.first << ")**-1 = " << test.second << std::endl; EXPECT(test.first.inverse() == test.second); LOG << "Test " << test.first << " = (" << test.second << ")**-1" << std::endl; EXPECT(test.first == test.second.inverse()); } Fraction zero(0); LOG << "Test (" << zero << ")**-1 (throw BadValue)" << std::endl; EXPECT_THROWS_AS(zero.inverse(), BadValue); } //---------------------------------------------------------------------------------------------------------------------- CASE("String to double to fraction to double to string") { Translator s2d; std::string s("-17.9229"); Fraction f(s2d(s)); std::ostringstream oss; oss << double(f); EXPECT(oss.str() == s); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { eckit::test::LOG.precision(16); return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/types/test_cache.cc0000664000175000017500000001116515161702250017725 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/Resource.h" #include "eckit/config/ResourceMgr.h" #include "eckit/container/Cache.h" #include "eckit/log/Log.h" #include "eckit/types/Types.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { struct Obj { Obj() : s_(), d_(0) {} Obj(const string& s, const unsigned long long& d) : s_(s), d_(d) {} string s_; unsigned long long d_; friend std::ostream& operator<<(std::ostream& s, const Obj& x) { s << x.s_ << ":" << x.d_; return s; } }; //---------------------------------------------------------------------------------------------------------------------- CASE("test_contructor") { Cache cache; EXPECT(cache.insert("a", Obj("aaa", 11111))); EXPECT(cache.insert("b", Obj("bbb", 22222))); EXPECT(cache.insert("c", Obj("ccc", 33333))); // cache.print(std::cout); EXPECT(cache.valid("a")); EXPECT(cache.valid("b")); EXPECT(cache.valid("c")); cache.clear(); EXPECT(!cache.valid("a")); EXPECT(!cache.valid("b")); EXPECT(!cache.valid("c")); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_expire") { Cache cache; EXPECT(cache.insert("a", Obj("aaa", 11111))); EXPECT(cache.insert("b", Obj("bbb", 22222))); EXPECT(cache.insert("c", Obj("ccc", 33333))); cache.expire("b"); // cache.print(std::cout); EXPECT(cache.size() == 3); EXPECT(cache.valid("a")); EXPECT(!cache.valid("b")); EXPECT(cache.valid("c")); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_purge") { Cache cache; EXPECT(cache.insert("a", Obj("aaa", 11111))); EXPECT(cache.insert("b", Obj("bbb", 22222))); EXPECT(cache.insert("c", Obj("ccc", 33333))); EXPECT(cache.insert("d", Obj("ddd", 44444))); EXPECT(cache.size() == 4); EXPECT(cache.expire("b")); EXPECT(cache.expire("d")); cache.purge(); cache.print(std::cout); EXPECT(cache.size() == 2); EXPECT(cache.valid("a")); EXPECT(!cache.valid("b")); EXPECT(cache.valid("c")); EXPECT(!cache.valid("d")); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_update") { Cache cache; EXPECT(cache.insert("a", Obj("aaa", 11111))); EXPECT(cache.insert("b", Obj("bbb", 22222))); EXPECT(cache.insert("c", Obj("ccc", 33333))); Obj o1("ddd", 44444); EXPECT(!cache.update("d", o1)); Obj o2("BBB", 2); EXPECT(cache.update("b", o2)); Obj o3; EXPECT(cache.fetch("d", o3)); EXPECT(o3.s_ == "ddd"); EXPECT(o3.d_ == 44444); EXPECT(cache.fetch("b", o3)); EXPECT(o3.s_ == "BBB"); EXPECT(o3.d_ == 2); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_fetch") { Cache cache; EXPECT(cache.insert("a", Obj("aaa", 11111))); EXPECT(cache.insert("b", Obj("bbb", 22222))); EXPECT(cache.insert("c", Obj("ccc", 33333))); EXPECT(cache.insert("d", Obj("ddd", 44444))); Obj o; EXPECT(!cache.fetch("f", o)); // no obj f exists EXPECT(cache.fetch("b", o)); // obj b exists EXPECT(o.s_ == "bbb"); EXPECT(o.d_ == 22222); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_insert") { Cache cache; EXPECT(cache.insert("a", Obj("aaa", 11111))); EXPECT(cache.insert("b", Obj("bbb", 22222))); EXPECT(cache.insert("c", Obj("ccc", 33333))); // double insert fails EXPECT(!cache.insert("a", Obj("AAA", 1))); Obj o; EXPECT(cache.fetch("a", o)); EXPECT(o.s_ != "AAA"); EXPECT(o.d_ != 1); EXPECT(o.s_ == "aaa"); EXPECT(o.d_ == 11111); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/types/CMakeLists.txt0000664000175000017500000000324415161702250020053 0ustar alastairalastairoption( ENABLE_ECKIT-395 "Control if ECKIT-395 tests should be enabled" ON ) # eckit_test_types_fixedstring known to fail compilation with PGI 19.4 (working with PGI 18.10) ecbuild_add_test( TARGET eckit_test_types_cache SOURCES test_cache.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_types_doublecompare SOURCES test_doublecompare.cc LIBS eckit_maths ) ecbuild_add_test( TARGET eckit_test_types_floatcompare SOURCES test_floatcompare.cc LIBS eckit_maths ) ecbuild_add_test( TARGET eckit_test_types_time SOURCES test_time.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_types_uuid SOURCES test_uuid.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_types_print_vector SOURCES test_print_vector.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_types_fixedstring SOURCES test_fixedstring.cc LIBS eckit CONDITION ENABLE_ECKIT-395 ) ecbuild_add_test( TARGET eckit_test_types_fraction SOURCES test_fraction.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_types_hour SOURCES test_hour.cc LIBS eckit ) # performance test ecbuild_add_test( TARGET eckit_test_types_double_compare_speed SOURCES test-double-compare-speed.cc LIBS eckit ) eckit-2.0.7/tests/types/test_floatcompare.cc0000664000175000017500000002632115161702250021336 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/maths/FloatingPointExceptions.h" #include "eckit/types/FloatCompare.h" #include "eckit/testing/Test.h" namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- namespace { bool is_equal(float a, float b, float epsilon, int maxUlps) { return types::is_approximately_equal(a, b, epsilon, maxUlps); } bool is_equal(float a, float b, float epsilon) { return types::is_approximately_equal(a, b, epsilon); } bool is_equal(float a, float b) { return types::is_approximately_equal(a, b, 0.00001f); } const float dEps = std::numeric_limits::epsilon(); const float dInf = std::numeric_limits::infinity(); const float sMin = std::numeric_limits::denorm_min(); const float dMin = std::numeric_limits::min(); const float dMax = std::numeric_limits::max(); const float qNaN = std::numeric_limits::quiet_NaN(); const float sNaN = std::numeric_limits::signaling_NaN(); }; // namespace //---------------------------------------------------------------------------------------------------------------------- CASE("test_large_numbers") { Log::info() << "test_large_numbers" << std::endl; EXPECT(is_equal(1000000, 1000000)); EXPECT(is_equal(1000000, 1000000.00001)); EXPECT(is_equal(1000000.00001, 1000000)); EXPECT(!is_equal(1000000.0, 1000001.0)); EXPECT(!is_equal(1000001.0, 1000000.0)); // ----------------------------------------------- EXPECT(is_equal(dMax, dMax)); EXPECT(is_equal(dMax, dMax, dEps)); EXPECT(is_equal(dMin, dMin)); EXPECT(is_equal(dMin, dMin, dEps)); } CASE("test_negative_large_numbers") { Log::info() << "test_negative_large_numbers " << dMin << std::endl; EXPECT(is_equal(-1000000, -1000000)); EXPECT(is_equal(-1000000, -1000000.00001)); EXPECT(is_equal(-1000000.00001, -1000000)); EXPECT(!is_equal(-1000000.0, -1000001.0)); EXPECT(!is_equal(-1000001.0, -1000000.0)); // ----------------------------------------------- EXPECT(is_equal(-dMax, -dMax)); EXPECT(is_equal(-dMax, -dMax, dEps)); EXPECT(is_equal(-dMin, -dMin)); EXPECT(is_equal(-dMin, -dMin, dEps)); } CASE("test_large_numbers_of_opposite_sign") { EXPECT(!is_equal(-1000000, 1000000)); EXPECT(!is_equal(-1000000, 1000000.00001)); EXPECT(!is_equal(-1000000.00001, 1000000)); EXPECT(!is_equal(-1000000.0, 1000001.0)); EXPECT(!is_equal(-1000001.0, 1000000.0)); // Overflow can occur here (as in CRAY) in eckit::types::is_approximately_equal maths::FloatingPointExceptions::disable_floating_point_exceptions(); EXPECT(!is_equal(-dMax, dMax)); EXPECT(!is_equal(-dMax, dMax, dEps)); maths::FloatingPointExceptions::enable_floating_point_exceptions(); } CASE("test_ulp_around_one") { Log::info() << "test_ulp_around_one" << std::endl; // ULP distances up to 10 are equal // Going right from 1 by eps increases distance by 1 // Going left from 1 by eps increases distance by 2 for (int i = 0; i <= 10; ++i) { EXPECT(is_equal(1.0 + i * dEps, 1.0, dEps)); EXPECT(is_equal(1.0, 1.0 + i * dEps, dEps)); EXPECT(is_equal(1.0 - i * dEps / 2, 1.0, dEps)); EXPECT(is_equal(1.0, 1.0 - i * dEps / 2, dEps)); } // ULP distances greater 10 are not equal EXPECT(!is_equal(1.0 + 11 * dEps, 1.0, dEps)); EXPECT(!is_equal(1.0, 1.0 + 11 * dEps, dEps)); EXPECT(!is_equal(1.0 - 11 * dEps / 2, 1.0, dEps)); EXPECT(!is_equal(1.0, 1.0 - 11 * dEps / 2, dEps)); } CASE("test_numbers_around_one") { Log::info() << "test_numbers_around_one" << std::endl; EXPECT(is_equal(1.0000001, 1.0000002)); EXPECT(is_equal(1.0000002, 1.0000001)); EXPECT(is_equal(1.123456, 1.123457)); EXPECT(is_equal(1.12345, 1.12344, 0.001)); EXPECT(!is_equal(1.0001, 1.0002)); EXPECT(!is_equal(1.0002, 1.0001)); } CASE("test_numbers_around_negative_one") { Log::info() << "test_numbers_around_negative_one" << std::endl; EXPECT(is_equal(-1.0000001, -1.0000002)); EXPECT(is_equal(-1.0000002, -1.0000001)); EXPECT(!is_equal(-1.0001, -1.0002)); EXPECT(!is_equal(-1.0002, -1.0001)); } CASE("test_numbers_between_one_and_zero") { Log::info() << "test_numbers_between_one_and_zero" << std::endl; EXPECT(is_equal(0.000000001000001, 0.000000001000002)); EXPECT(is_equal(0.000000001000002, 0.000000001000001)); EXPECT(!is_equal(0.0012, 0.0011)); EXPECT(!is_equal(0.0011, 0.0012)); } CASE("test_numbers_between_minusone_and_zero") { Log::info() << "test_numbers_between_minusone_and_zero" << std::endl; EXPECT(is_equal(-0.000000001000001, -0.000000001000002)); EXPECT(is_equal(-0.000000001000002, -0.000000001000001)); EXPECT(!is_equal(-0.0012, -0.0011)); EXPECT(!is_equal(-0.0011, -0.0012)); } CASE("test_comparisons_involving_zero") { Log::info() << "test_comparisons_involving_zero" << std::endl; EXPECT(is_equal(0.0, 0.0)); EXPECT(is_equal(0.0, -0.0)); EXPECT(is_equal(-0.0, -0.0)); EXPECT(!is_equal(0.0001, 0.0)); EXPECT(!is_equal(0.0, 0.0001)); EXPECT(!is_equal(-0.0001, 0.0)); EXPECT(!is_equal(0.0, -0.0001)); EXPECT(is_equal(0.0, 1e-30, 0.01)); EXPECT(is_equal(1e-30, 0.0, 0.01)); EXPECT(!is_equal(1e-30, 0.0, 1e-31)); EXPECT(!is_equal(0.0, 1e-30, 1e-31)); EXPECT(is_equal(0.0, -1e-30, 0.1)); EXPECT(is_equal(-1e-30, 0.0, 0.1)); EXPECT(!is_equal(-1e-30, 0.0, 1e-31)); EXPECT(!is_equal(0.0, -1e-30, 1e-31)); } CASE("test_comparisons_involving_infinity") { Log::info() << "test_comparisons_involving_infinity" << std::endl; if (std::numeric_limits::has_infinity) { EXPECT(is_equal(dInf, dInf)); EXPECT(is_equal(-dInf, -dInf)); EXPECT(!is_equal(dInf, dMax)); EXPECT(!is_equal(dMax, dInf)); EXPECT(!is_equal(-dInf, -dMax)); EXPECT(!is_equal(-dMax, -dInf)); } else { Log::info() << "test_comparisons_involving_infinity NOT VALID on this platform" << std::endl; } } CASE("test_comparisons_involving_nan") { Log::info() << "test_comparisons_involving_nan" << std::endl; // The value NaN (Not a Number) is used to represent a value that does not represent a real number. // NaN's are represented by a bit pattern with an exponent of all 1s and a non-zero fraction. T // there are two categories of NaN: QNaN (Quiet NaN) and SNaN (Signalling NaN). // // A QNaN is a NaN with the most significant fraction bit set. // QNaN's propagate freely through most arithmetic operations. // These values pop out of an operation when the result is not mathematically defined. // An SNaN is a NaN with the most significant fraction bit clear. // It is used to signal an exception when used in operations. // SNaN's can be handy to assign to uninitialized variables to trap premature usage. // Semantically, QNaN's denote indeterminate operations, while SNaN's denote invalid operations. EXPECT(!is_equal(qNaN, qNaN)); EXPECT(!is_equal(qNaN, 0.0)); EXPECT(!is_equal(-0.0, qNaN)); EXPECT(!is_equal(qNaN, -0.0)); EXPECT(!is_equal(0.0, qNaN)); EXPECT(!is_equal(qNaN, dInf)); EXPECT(!is_equal(dInf, qNaN)); EXPECT(!is_equal(qNaN, dMax)); EXPECT(!is_equal(dMax, qNaN)); EXPECT(!is_equal(qNaN, -dMax)); EXPECT(!is_equal(-dMax, qNaN)); EXPECT(!is_equal(qNaN, dMin)); EXPECT(!is_equal(dMin, qNaN)); EXPECT(!is_equal(qNaN, -dMin)); EXPECT(!is_equal(-dMin, qNaN)); maths::FloatingPointExceptions::disable_floating_point_exceptions(); EXPECT(!is_equal(sNaN, sNaN)); EXPECT(!is_equal(sNaN, 0.0)); EXPECT(!is_equal(-0.0, sNaN)); EXPECT(!is_equal(sNaN, -0.0)); EXPECT(!is_equal(0.0, sNaN)); EXPECT(!is_equal(sNaN, dInf)); EXPECT(!is_equal(dInf, sNaN)); EXPECT(!is_equal(sNaN, dMax)); EXPECT(!is_equal(dMax, sNaN)); EXPECT(!is_equal(sNaN, -dMax)); EXPECT(!is_equal(-dMax, sNaN)); EXPECT(!is_equal(sNaN, dMin)); EXPECT(!is_equal(dMin, sNaN)); EXPECT(!is_equal(sNaN, -dMin)); EXPECT(!is_equal(-dMin, sNaN)); maths::FloatingPointExceptions::enable_floating_point_exceptions(); } CASE("test_comparisons_opposite_side_of_zero") { Log::info() << "test_comparisons_opposite_side_of_zero" << std::endl; EXPECT(!is_equal(1.0000001, -1.0)); EXPECT(!is_equal(-1.0, 1.0000001)); EXPECT(!is_equal(-1.0000001, 1.0)); EXPECT(!is_equal(1.0, -1.0000001)); EXPECT(is_equal(10.0 * dMin, 10.0 * -dMin)); EXPECT(is_equal(10000 * dMin, 10000 * -dMin)); } CASE("test_comparisons_very_close_to_zero") { Log::info() << "test_comparisons_very_close_to_zero" << std::endl; EXPECT(is_equal(dMin, -dMin, dEps)); EXPECT(is_equal(-dMin, dMin, dEps)); EXPECT(is_equal(dMin, 0, dEps)); EXPECT(is_equal(0, dMin, dEps)); EXPECT(is_equal(-dMin, 0, dEps)); EXPECT(is_equal(0, -dMin, dEps)); EXPECT(is_equal(0.000000001, -dMin)); EXPECT(is_equal(0.000000001, dMin)); EXPECT(is_equal(dMin, 0.000000001)); EXPECT(is_equal(-dMin, 0.000000001)); EXPECT(!is_equal(0.000000001, -dMin, 1e-10)); EXPECT(!is_equal(0.000000001, dMin, 1e-10)); EXPECT(!is_equal(dMin, 0.000000001, 1e-10)); EXPECT(!is_equal(-dMin, 0.000000001, 1e-10)); } CASE("test_comparisons_with_denormal_numbers") { Log::info() << "test_comparisons_with_denormal_numbers" << std::endl; EXPECT(is_equal(sMin, -sMin, dEps)); EXPECT(is_equal(-sMin, sMin, dEps)); EXPECT(is_equal(sMin, 0, dEps)); EXPECT(is_equal(0, sMin, dEps)); EXPECT(is_equal(-sMin, 0, dEps)); EXPECT(is_equal(0, -sMin, dEps)); const float lMin = dMin - sMin; // largest denormal number EXPECT(is_equal(lMin, -lMin, dEps)); EXPECT(is_equal(-lMin, lMin, dEps)); EXPECT(is_equal(lMin, 0, dEps)); EXPECT(is_equal(0, lMin, dEps)); EXPECT(is_equal(-lMin, 0, dEps)); EXPECT(is_equal(0, -lMin, dEps)); } CASE("test_comparisons_ulps") { Log::info() << "test_comparisons_ulps" << std::endl; EXPECT(is_equal(dMin, -dMin, 0, 2)); EXPECT(is_equal(-dMin, dMin, 0, 2)); EXPECT(is_equal(dMin, 0, 0, 1)); EXPECT(is_equal(0, dMin, 0, 1)); EXPECT(is_equal(-dMin, 0, 0, 1)); EXPECT(is_equal(0, -dMin, 0, 1)); EXPECT(!is_equal(dMin, -dMin, 0, 1)); EXPECT(!is_equal(-dMin, dMin, 0, 1)); EXPECT(!is_equal(dMin, 0, 0, 0)); EXPECT(!is_equal(0, dMin, 0, 0)); EXPECT(!is_equal(-dMin, 0, 0, 0)); EXPECT(!is_equal(0, -dMin, 0, 0)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { eckit::maths::FloatingPointExceptions::enable_floating_point_exceptions(); return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/log/0000775000175000017500000000000015161702250014725 5ustar alastairalastaireckit-2.0.7/tests/log/test_log_user_channels.cc0000664000175000017500000000257515161702250021776 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor * does it submit to any jurisdiction. */ #include #include "eckit/log/Channel.h" #include "eckit/log/ColouringTarget.h" #include "eckit/log/OStreamTarget.h" #include "eckit/runtime/Main.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit { namespace test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_user_log_channel_registration") { // FIXME: re-enable this test // EXPECT_NO_THROW( Log::registerChannel("mychannel", new Channel(new ColouringTarget(new // OStreamTarget(std::cout), &Colour::green))) ); // EXPECT_NO_THROW( Log::channel("mychannel") << "TEST MY VERY OWN CHANNEL" << std::endl ); } //---------------------------------------------------------------------------------------------------------------------- } // namespace test } // namespace eckit int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/log/test_colour.cc0000664000175000017500000000273715161702250017607 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/Colour.h" #include "eckit/log/Log.h" #include "eckit/runtime/Tool.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_colour") { Log::info() << "Hello, worlds" << std::endl; std::cout << Colour::red << "Red" << Colour::reset << std::endl; std::cout << Colour::off; std::cout << Colour::red << "Red" << Colour::reset << std::endl; std::cout << Colour::on; std::cout << Colour::blue << "Red" << Colour::reset << std::endl; std::cout << Colour::red << Colour::bold << "Red" << Colour::reset << std::endl; std::cout << Colour::yellow << Colour::underline << "Red" << Colour::reset << std::endl; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/log/test_log_channels.cc0000664000175000017500000001142715161702250020734 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/os/BackTrace.h" #include "eckit/runtime/Tool.h" #include "eckit/log/CallbackTarget.h" #include "eckit/log/Channel.h" #include "eckit/log/ColouringTarget.h" #include "eckit/log/FileTarget.h" #include "eckit/log/OStreamTarget.h" #include "eckit/log/WrapperTarget.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; //---------------------------------------------------------------------------------------------------------------------- #if 1 #define DEBUG_H #define DEBUG_(x) #else #define DEBUG_H std::cerr << " DEBUG @ " << __FILE__ << " +" << __LINE__ << std::endl; #define DEBUG_(x) std::cerr << #x << " : [" << x << "] @ " << __FILE__ << " +" << __LINE__ << std::endl; #endif namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- /// Example of a wrapper target that capitalizes all output class CapitalizerTarget : public WrapperTarget { public: CapitalizerTarget(LogTarget* target) : WrapperTarget(target) {} private: virtual void write(const char* start, const char* end) { std::string::size_type length = std::distance(start, end); buffer_.resize(length); std::locale loc; const char* p = start; for (std::string::size_type i = 0; i < length; ++i, ++p) { buffer_[i] = std::toupper(*p, loc); } target_->write(buffer_.c_str(), buffer_.c_str() + length); } virtual void writePrefix() {} virtual void writeSuffix() {} void print(std::ostream& s) const { s << "CapitalizerTarget()"; } std::string buffer_; }; //---------------------------------------------------------------------------------------------------------------------- static void callback_ctxt(void* ctxt, const char* msg) { std::cout << "[" << *((int*)ctxt) << "] : -- " << msg << std::endl; } static void callback_noctxt(void*, const char* msg) { std::cout << "[CALLBACK OUT] : -- " << msg << std::endl; } //---------------------------------------------------------------------------------------------------------------------- CASE("test_multi_targets") { std::cout << "---> test_multi_targets()" << std::endl; int t = 0; Channel mychannel; mychannel << "testing [" << t++ << "]" << std::endl; mychannel.addFile("test.txt"); mychannel << "testing [" << t++ << "]" << std::endl; std::ofstream of("test.txt.2"); mychannel.addStream(of); mychannel << "testing [" << t++ << "]" << std::endl; mychannel.addStream(std::cout); mychannel << "testing [" << t++ << "]" << std::endl; mychannel.addStream(std::cerr); mychannel << "testing [" << t++ << "]" << std::endl; std::ostringstream oss; mychannel.addStream(oss); mychannel << "testing [" << t++ << "]" << std::endl; mychannel.addCallback(&callback_noctxt, nullptr); mychannel.addCallback(&callback_ctxt, &t); mychannel << "testing [" << t++ << "]" << std::endl; // mychannel.addLogTarget(new CapitalizerTarget(new FileTarget(PathName("capitals.txt")))); mychannel << "testing [" << t++ << "]" << std::endl; mychannel << "Final test" << std::endl; } CASE("test_multi_colouring") { #if 0 Log::info().setLogTarget( new ColouringTarget(new OStreamTarget(std::cout), &Colour::green)); Log::warning().setLogTarget( new ColouringTarget(new OStreamTarget(std::cerr), &Colour::yellow)); Log::error().setLogTarget( new ColouringTarget(new OStreamTarget(std::cerr), &Colour::red)); Log::info() << "Log::info() is green" << std::endl; Log::warning() << "Log::warning() is yellow" << std::endl; Log::error() << "Log::error() is red" << std::endl; #endif } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test void on_signal_dumpbacktrace(int signum) { printf("Caught signal %d\n", signum); std::cerr << BackTrace::dump() << std::endl; eckit::LibEcKit::instance().abort(); } int main(int argc, char** argv) { signal(SIGSEGV, on_signal_dumpbacktrace); return run_tests(argc, argv); } eckit-2.0.7/tests/log/test_log_json.cc0000664000175000017500000001110615161702250020104 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/LibEcKit.h" #include "eckit/log/JSON.h" #include "eckit/log/Log.h" #include "eckit/runtime/Tool.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- void write_json(eckit::JSON& json) { json.startObject(); json << "a" << 1; json << "b"; { json.startObject(); json << "b1" << 2; json << "b2" << 3; json.endObject(); } json << "c" << 4; json << "d"; { json.startList(); json << 1 << 2 << 3 << 4 << 5; json.endList(); } json.endObject(); } CASE("compact (default)") { std::stringstream s; JSON json(s); write_json(json); EXPECT(s.str() == "{\"a\":1,\"b\":{\"b1\":2,\"b2\":3},\"c\":4,\"d\":[1,2,3,4,5]}"); } CASE("compact (explicit)") { JSON::Formatting formatting(JSON::Formatting::COMPACT); std::stringstream s; JSON json(s, formatting); write_json(json); EXPECT(s.str() == "{\"a\":1,\"b\":{\"b1\":2,\"b2\":3},\"c\":4,\"d\":[1,2,3,4,5]}"); } CASE("indent(2) --> INDENT_DICT, indent=2") { std::stringstream s; JSON json(s, JSON::Formatting::indent(2)); write_json(json); EXPECT(s.str() == "{\n" " \"a\" : 1,\n" " \"b\" : {\n" " \"b1\" : 2,\n" " \"b2\" : 3\n" " },\n" " \"c\" : 4,\n" " \"d\" : [1,2,3,4,5]\n" "}"); } CASE("indent(4) --> INDENT_DICT, indent=4") { std::stringstream s; JSON json(s, JSON::Formatting::indent(4)); write_json(json); EXPECT(s.str() == "{\n" " \"a\" : 1,\n" " \"b\" : {\n" " \"b1\" : 2,\n" " \"b2\" : 3\n" " },\n" " \"c\" : 4,\n" " \"d\" : [1,2,3,4,5]\n" "}"); } CASE("INDENT_LIST, indent=2") { JSON::Formatting formatting(JSON::Formatting::INDENT_LIST); std::stringstream s; JSON json(s, formatting); write_json(json); EXPECT(s.str() == "{\"a\" : 1,\"b\" : {\"b1\" : 2,\"b2\" : 3},\"c\" : 4,\"d\" : [\n" " 1,\n" " 2,\n" " 3,\n" " 4,\n" " 5\n" "]}"); } CASE("INDENT_DICT, indent=2") { JSON::Formatting formatting(JSON::Formatting::INDENT_DICT); std::stringstream s; JSON json(s, formatting); write_json(json); EXPECT(s.str() == "{\n" " \"a\" : 1,\n" " \"b\" : {\n" " \"b1\" : 2,\n" " \"b2\" : 3\n" " },\n" " \"c\" : 4,\n" " \"d\" : [1,2,3,4,5]\n" "}"); } CASE("INDENT_ALL, indent=2") { JSON::Formatting formatting(JSON::Formatting::INDENT_ALL); std::stringstream s; JSON json(s, formatting); write_json(json); EXPECT(s.str() == "{\n" " \"a\" : 1,\n" " \"b\" : {\n" " \"b1\" : 2,\n" " \"b2\" : 3\n" " },\n" " \"c\" : 4,\n" " \"d\" : [\n" " 1,\n" " 2,\n" " 3,\n" " 4,\n" " 5\n" " ]\n" "}"); } CASE("INDENT_DICT | INDENT_LIST, indent=4") { JSON::Formatting formatting(JSON::Formatting::INDENT_DICT | JSON::Formatting::INDENT_LIST, 4); std::stringstream s; JSON json(s, formatting); write_json(json); EXPECT(s.str() == "{\n" " \"a\" : 1,\n" " \"b\" : {\n" " \"b1\" : 2,\n" " \"b2\" : 3\n" " },\n" " \"c\" : 4,\n" " \"d\" : [\n" " 1,\n" " 2,\n" " 3,\n" " 4,\n" " 5\n" " ]\n" "}"); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/log/test_syslog.cc0000664000175000017500000000643115161702250017617 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/SysLog.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_priority") { SysLog log("Test message", 0, SysLog::Local7, SysLog::Info); std::string logString = static_cast(log); std::string expectedPriority = "<" + std::to_string(log.priority()) + ">"; EXPECT(logString.find(expectedPriority) != std::string::npos); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_timezone") { SysLog log("Test message", 0, SysLog::Local7, SysLog::Info); std::string logString = static_cast(log); // Check if 'Z' UTC indicator is present EXPECT(logString.find("Z") != std::string::npos); // Check if 'T' separator is present EXPECT(logString.find("T") != std::string::npos); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_appname") { SysLog log("Test message", 0, SysLog::Local7, SysLog::Info); EXPECT(log.appName() == Main::instance().name()); // Change the appName and check if it persists log.appName("test_app"); std::string logString = static_cast(log); EXPECT(logString.find("test_app") != std::string::npos); // Create a new SysLog instance and check if it retains the original appName SysLog newLog("New message", 2, SysLog::Local7, SysLog::Info); EXPECT(newLog.appName() == Main::instance().name()); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_without_structured_data") { SysLog log("Test message", 0, SysLog::Local7, SysLog::Info); std::string logString = static_cast(log); // Check if structured data is not present EXPECT(logString.find("[origin") == std::string::npos); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_with_structured_data") { SysLog log("Test message", 0, SysLog::Local7, SysLog::Info); log.software("log_test"); log.swVersion("1.0.0"); log.enterpriseId("7464"); std::string logString = static_cast(log); EXPECT(logString.find("enterpriseId=\"7464\"") != std::string::npos); EXPECT(logString.find("software=\"log_test\"") != std::string::npos); EXPECT(logString.find("swVersion=\"1.0.0\"") != std::string::npos); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/log/test_log_tcp.cc0000664000175000017500000001364015161702250017726 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file test_log_tcp.cc /// @author Metin Cakircali /// @date Jun 2024 #include #include #include #include #include #include #include #include #include "eckit/io/SockBuf.h" #include "eckit/log/Channel.h" #include "eckit/log/Log.h" #include "eckit/log/SysLog.h" #include "eckit/log/SysLogTCPTarget.h" #include "eckit/net/TCPClient.h" #include "eckit/net/TCPServer.h" #include "eckit/net/TCPStream.h" #include "eckit/runtime/Main.h" #include "eckit/testing/Test.h" using namespace eckit; //---------------------------------------------------------------------------------------------------------------------- // HELPERS namespace { const std::array test_messages = {"Hello, World!", "Logging over TCP", "This is a test message", "Another log entry", "Testing TCP logging", "Log message number 6", "Seventh message in the log", "Eighth log entry", "Ninth message for testing", "Some more log messages", "Yet another log message", "Final log message for this test", "End of test messages", "Logging complete", "All messages sent successfully", "TCP logging test finished", "No errors encountered", "Test messages are consistent", "This is a very long log message to test the TCP logging for long " "messages, which should not cause any issues or truncation. ", "TCP connection is stable", "Server received all messages", "Client sent all messages", "Test completed without issues", "Ready for next test"}; //---------------------------------------------------------------------------------------------------------------------- // a simple server that listens for incoming messages over TCP and checks them against expected messages void listenLogs(int port) { net::TCPServer server(port); auto socket = server.accept(); SockBuf buffer(socket); std::istream ins(&buffer); for (const auto& msg : test_messages) { std::string received; std::getline(ins, received); EXPECT_EQUAL(received, msg); } } void listenSysLogs(int port) { net::TCPServer server(port); auto socket = server.accept(); SockBuf buffer(socket); std::istream ins(&buffer); for (const auto& msg : test_messages) { std::string received; std::getline(ins, received); // std::cout << "Received: " << received << std::endl; EXPECT(received.find(msg) != std::string::npos); EXPECT(received.find("software=\"test-eckit\"") != std::string::npos); EXPECT(received.find("swVersion=\"0.0.1\"") != std::string::npos); } } //---------------------------------------------------------------------------------------------------------------------- // send logs over TCP void sendLogs(const int port) { auto socket = net::TCPClient().connect("localhost", port); Log::info().setTarget(new TCPTarget(socket)); for (const auto& msg : test_messages) { Log::info() << msg << '\n'; } // reset target to default Log::info().setTarget(Main::instance().createInfoLogTarget()); } // send sysLogs over TCP void sendSysLogs(const int port) { SysLog log("", 0, SysLog::User, SysLog::Info); log.enterpriseId("7464"); // ECMWF's id log.software("test-eckit"); log.swVersion("0.0.1"); auto socket = net::TCPClient().connect("localhost", port); Log::info().setTarget(new SysLogTCPTarget(socket, log)); for (const auto& msg : test_messages) { Log::info() << msg << std::endl; } // reset target to default Log::info().setTarget(Main::instance().createInfoLogTarget()); } } // namespace //---------------------------------------------------------------------------------------------------------------------- namespace eckit::test { CASE("Log: send logs over TCP") { constexpr int test_port = 7575; std::thread server(listenLogs, test_port); std::thread client(sendLogs, test_port); client.join(); server.join(); } CASE("Log: send syslogs over TCP") { constexpr int test_port = 7576; std::thread server(listenSysLogs, test_port); std::thread client(sendSysLogs, test_port); client.join(); server.join(); } } // namespace eckit::test //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { return testing::run_tests(argc, argv); } eckit-2.0.7/tests/log/test_log_callback.cc0000664000175000017500000000510615161702250020672 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/filesystem/LocalPathName.h" #include "eckit/log/Log.h" #include "eckit/os/BackTrace.h" #include "eckit/runtime/Main.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- struct CTxt { std::string name_; }; static void output_callback_noctxt(void* ctxt, const char* msg) { // std::cout << "[DEBUG] -- " << Here() << "\n" << BackTrace::dump() << "\n" << std::endl; std::cout << "[FORWARD OUT] -- " << msg; } static void output_callback_withctxt(void* ctxt, const char* msg) { // std::cout << "[DEBUG] -- " << Here() << "\n" << BackTrace::dump() << "\n" << std::endl; std::cout << "[FORWARD OUT] -- CTXT [" << static_cast(ctxt)->name_ << "] -- " << msg; } //---------------------------------------------------------------------------------------------------------------------- /// tests without callback CASE("test_callback_none") { Log::info().clear(); Log::info() << "info message 1" << std::endl; Log::warning() << "warning message 1" << std::endl; Log::error() << "error message 1" << std::endl; Log::info().reset(); } /// tests with null context CASE("test_callback_noctxt") { Log::info().setCallback(&output_callback_noctxt); Log::info() << "info message 1" << std::endl; Log::warning() << "warning message 1" << std::endl; Log::error() << "error message 1" << std::endl; Log::info().clear(); Log::info().reset(); } /// tests with context CASE("test_callback_withctxt") { CTxt ctxt; ctxt.name_ = "MyTest"; Log::info().setCallback(&output_callback_withctxt, &ctxt); Log::info() << "info message 1" << std::endl; Log::warning() << "warning message 1" << std::endl; Log::error() << "error message 1" << std::endl; Log::info().clear(); Log::info().reset(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/log/CMakeLists.txt0000664000175000017500000000250015161702250017462 0ustar alastairalastairecbuild_add_test( TARGET eckit_test_log SOURCES test_log.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_log_channels SOURCES test_log_channels.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_log_callback SOURCES test_log_callback.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_log_threads SOURCES test_log_threads.cc ENVIRONMENT _TEST_ECKIT_HOME=/tmp/$ENV{USER} LIBS eckit ) ecbuild_add_test( TARGET eckit_test_log_colour SOURCES test_colour.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_log_json SOURCES test_log_json.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_log_user_channels ENABLED OFF SOURCES test_log_user_channels.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_syslog SOURCES test_syslog.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_log_tcp SOURCES test_log_tcp.cc LIBS eckit ) eckit-2.0.7/tests/log/test_log.cc0000664000175000017500000001034315161702250017055 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/config/LibEcKit.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/log/Seconds.h" #include "eckit/runtime/Tool.h" #include "eckit/types/DateTime.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_debug") { Log::debug() << "debug message 1" << std::endl; } CASE("test_debug_library") { Log::debug() << "debug message 2" << std::endl; } CASE("test_debug_macro") { LOG_DEBUG(true, LibEcKit) << "debug message 3" << std::endl; LOG_DEBUG(false, LibEcKit) << "debug message 4" << std::endl; } CASE("test_info") { Log::info() << "info message 1" << std::endl; } CASE("test_warning") { Log::warning() << "warning message 1" << std::endl; } CASE("test_error") { Log::error() << "error message 1" << std::endl; } CASE("test_panic") { Log::panic() << "panic message 1" << std::endl; } CASE("test_strerr") { LocalPathName p("/tmp/edfpmjq3480hfnsribnzasdfibv"); p.unlink(); } CASE("test_bytes") { eckit::Bytes b(1.); EXPECT("1 byte" == std::string(b)); EXPECT("1 " == b.shorten()); b = -1.; EXPECT("-1 byte" == std::string(b)); EXPECT("-1 " == b.shorten()); b = 1024.; EXPECT("1 Kbyte" == std::string(b)); EXPECT("1K" == b.shorten()); b = 1024. * 1024.; EXPECT("1 Mbyte" == std::string(b)); EXPECT("1M" == b.shorten()); b = 1024. * 1024. * 1024.; EXPECT("1 Gbyte" == std::string(b)); EXPECT("1G" == b.shorten()); b = 1024. * 1024. * 1024. * 1024.; EXPECT("1 Tbyte" == std::string(b)); EXPECT("1T" == b.shorten()); b = 1024. * 1024. * 1024. * 1024. * 1024.; EXPECT("1 Pbyte" == std::string(b)); EXPECT("1P" == b.shorten()); b = 1024. * 1024. * 1024. * 1024. * 1024. * 1024.; EXPECT("1 Ebyte" == std::string(b)); EXPECT("1E" == b.shorten()); b = 1024. * 1024. * 1024. * 1024. * 1024. * 1024. * 1024.; EXPECT("1 Zbyte" == std::string(b)); EXPECT("1Z" == b.shorten()); b = 1024. * 1024. * 1024. * 1024. * 1024. * 1024. * 1024. * 1024.; EXPECT("1 Ybyte" == std::string(b)); EXPECT("1Y" == b.shorten()); b = 1024. * 1024. * 1024. * 1024. * 1024. * 1024. * 1024. * 1024. * 1024.; EXPECT("1024 Ybytes" == std::string(b)); EXPECT("99Y" == b.shorten()); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_seconds") { { double time = 4.0 + (1.0 / 3.0); eckit::Seconds s{time * 3600.}; EXPECT_EQUAL("4 hours 20 minutes", std::string(s)); eckit::Seconds sc{time * 3600., true}; EXPECT_EQUAL("4h20m0s", std::string(sc)); } { eckit::Seconds s{10.4}; EXPECT_EQUAL("10 seconds", std::string(s)); eckit::Seconds sc{10.4, true}; EXPECT_EQUAL("10s", std::string(sc)); } } CASE("test_datetime") { double time = 4.0 + (1.0 / 3.0); { eckit::DateTime dt(Date(2016, 3, 31), Time(0, 0, 0)); EXPECT_EQUAL("2016-03-31T00:00:00Z", dt.iso(true)); dt = dt + eckit::Second{time * 3600.}; EXPECT_EQUAL("2016-03-31T04:20:00Z", dt.iso(true)); } { eckit::DateTime dt(Date(2016, 3, 31), Time(12, 0, 0, true)); dt = dt + eckit::Second{12 * 3600.}; EXPECT_EQUAL("2016-04-01T00:00:00Z", dt.iso(true)); } { eckit::DateTime dt{Date{2016, 3, 31}, Second{10.4}}; EXPECT_EQUAL("2016-03-31T00:00:10Z", dt.iso(true)); dt = dt + eckit::Second{0.4}; EXPECT_EQUAL("2016-03-31T00:00:11Z", dt.iso(true)); } } } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/log/test_log_threads.cc0000664000175000017500000000527715161702250020601 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/filesystem/LocalPathName.h" #include "eckit/log/CallbackTarget.h" #include "eckit/log/Channel.h" #include "eckit/log/Log.h" #include "eckit/os/BackTrace.h" #include "eckit/runtime/Tool.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" #include "eckit/thread/Thread.h" #include "eckit/thread/ThreadControler.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- static StaticMutex local_mutex; static void callback_logger(void* ctxt, const char* msg) { AutoLock lock(local_mutex); ///< usually global resources like this need to be protected by mutex std::cout << "[TEST] -- " << msg; } static void callback_special(void* ctxt, const char* msg) { std::cout << ">>> " << msg; } //---------------------------------------------------------------------------------------------------------------------- #define LOOPS 3 #define WAIT 10000 template class TLog : public Thread { void run() { // this shows how you can change the callback per thread if (N == 2) { Log::info().setCallback(&callback_special); } for (int i = 0; i < LOOPS * N; ++i) { ::usleep(N * WAIT); Log::info() << "thread [" << N << "] -- " << i << std::endl; } Log::info() << "thread [" << N << "] -- done !" << std::endl; } }; //---------------------------------------------------------------------------------------------------------------------- CASE("test_log_threads") { Log::info().setCallback(&callback_logger); Log::info() << ">>> starting ... " << std::endl; ThreadControler t1(new TLog<1>(), false); ThreadControler t2(new TLog<2>(), false); ThreadControler t3(new TLog<3>(), false); t1.start(); t2.start(); t3.start(); t1.wait(); t2.wait(); t3.wait(); Log::info() << ">>> finished!" << std::endl; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/config/0000775000175000017500000000000015161702250015411 5ustar alastairalastaireckit-2.0.7/tests/config/test_configuration.cc0000664000175000017500000004142215161702250021631 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/LocalConfiguration.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/filesystem/PathName.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" #include "eckit/utils/Hash.h" namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_configuration_interface") { bool value_bool = true; int value_int = 1; long value_long = 2; long long value_long_long = 2; size_t value_size_t = 3; float value_float = 1.234567; double value_double = 1.2345678912345789123456789; std::string value_string = "string"; std::vector value_arr_int = {1, 2, 3}; std::vector value_arr_long = {4, 5}; std::vector value_arr_long_long = {4, 5}; std::vector value_arr_size_t = {6, 7}; std::vector value_arr_float = {1.234567, 2.345678}; std::vector value_arr_double = {1.234567, 2.345678}; std::vector value_arr_string = {"hello", "world"}; std::int32_t value_int32 = value_int; std::int64_t value_int64 = value_long_long; std::vector value_arr_int32 = {4, 5}; std::vector value_arr_int64 = {4, 5}; bool result_bool = false; int result_int = 0; long result_long = 0; long long result_long_long = 0; size_t result_size_t = 0; float result_float = 0; double result_double = 0; std::string result_string; std::vector result_arr_int; std::vector result_arr_long; std::vector result_arr_long_long; std::vector result_arr_size_t; std::vector result_arr_float; std::vector result_arr_double; std::vector result_arr_string; std::int32_t result_int32 = 0; std::int64_t result_int64 = 0; std::vector result_arr_int32; std::vector result_arr_int64; LocalConfiguration local; { local.set("bool", value_bool); local.set("int", value_int); local.set("long", value_long); local.set("long long", value_long_long); local.set("size_t", value_size_t); local.set("float", value_float); local.set("double", value_double); local.set("string", value_string); local.set("arr_int", value_arr_int); local.set("arr_long", value_arr_long); local.set("arr_long_long", value_arr_long_long); local.set("arr_size_t", value_arr_size_t); local.set("arr_float", value_arr_float); local.set("arr_double", value_arr_double); local.set("arr_string", value_arr_string); local.set("int32", value_int32); local.set("int64", value_int64); local.set("arr_int32", value_arr_int32); local.set("arr_int64", value_arr_int64); } const Configuration& conf = local; EXPECT(!conf.get("missing", result_bool)); EXPECT(!conf.get("missing", result_int)); EXPECT(!conf.get("missing", result_long)); EXPECT(!conf.get("missing", result_long_long)); EXPECT(!conf.get("missing", result_size_t)); EXPECT(!conf.get("missing", result_float)); EXPECT(!conf.get("missing", result_double)); EXPECT(!conf.get("missing", result_string)); EXPECT(!conf.get("missing", result_arr_int)); EXPECT(!conf.get("missing", result_arr_long)); EXPECT(!conf.get("missing", result_arr_long_long)); EXPECT(!conf.get("missing", result_arr_size_t)); EXPECT(!conf.get("missing", result_arr_float)); EXPECT(!conf.get("missing", result_arr_double)); EXPECT(!conf.get("missing", result_arr_string)); EXPECT(!conf.get("missing", result_int32)); EXPECT(!conf.get("missing", result_int64)); EXPECT(!conf.get("missing", result_arr_int32)); EXPECT(!conf.get("missing", result_arr_int64)); EXPECT(conf.get("bool", result_bool)); EXPECT(conf.get("int", result_int)); EXPECT(conf.get("long", result_long)); EXPECT(conf.get("long long", result_long_long)); EXPECT(conf.get("size_t", result_size_t)); EXPECT(conf.get("float", result_float)); EXPECT(conf.get("double", result_double)); EXPECT(conf.get("string", result_string)); EXPECT(conf.get("arr_int", result_arr_int)); EXPECT(conf.get("arr_long", result_arr_long)); EXPECT(conf.get("arr_long_long", result_arr_long_long)); EXPECT(conf.get("arr_size_t", result_arr_size_t)); EXPECT(conf.get("arr_float", result_arr_float)); EXPECT(conf.get("arr_double", result_arr_double)); EXPECT(conf.get("arr_string", result_arr_string)); EXPECT(conf.get("int32", result_int32)); EXPECT(conf.get("int64", result_int64)); EXPECT(conf.get("arr_int32", result_arr_int32)); EXPECT(conf.get("arr_int64", result_arr_int64)); EXPECT(result_bool == value_bool); EXPECT(result_int == value_int); EXPECT(result_long == value_long); EXPECT(result_long_long == value_long_long); EXPECT(result_size_t == value_size_t); EXPECT(result_float == value_float); EXPECT(result_double == value_double); EXPECT(result_string == value_string); EXPECT(result_arr_int == value_arr_int); EXPECT(result_arr_long_long == value_arr_long_long); EXPECT(result_arr_size_t == value_arr_size_t); EXPECT(result_arr_float == value_arr_float); EXPECT(result_arr_double == value_arr_double); EXPECT(result_arr_string == value_arr_string); EXPECT(result_arr_int64 == value_arr_int64); EXPECT(conf.getBool("bool") == value_bool); EXPECT(conf.getInt("int") == value_int); EXPECT(conf.getLong("long") == value_long); EXPECT(conf.getLong("long long") == value_long_long); EXPECT(conf.getUnsigned("size_t") == value_size_t); EXPECT(conf.getFloat("float") == value_float); EXPECT(conf.getDouble("double") == value_double); EXPECT(conf.getString("string") == value_string); EXPECT(conf.getIntVector("arr_int") == value_arr_int); EXPECT(conf.getLongVector("arr_long") == value_arr_long); EXPECT(conf.getUnsignedVector("arr_size_t") == value_arr_size_t); EXPECT(conf.getFloatVector("arr_float") == value_arr_float); EXPECT(conf.getDoubleVector("arr_double") == value_arr_double); EXPECT(conf.getStringVector("arr_string") == value_arr_string); EXPECT(conf.getInt32("int32") == value_int32); EXPECT(conf.getInt64("int64") == value_int64); EXPECT(conf.getInt32Vector("arr_int32") == value_arr_int32); EXPECT(conf.getInt64Vector("arr_int64") == value_arr_int64); // Get throwing EXPECT_THROWS_AS(conf.getBool("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getInt("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getLong("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getUnsigned("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getFloat("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getDouble("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getString("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getIntVector("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getLongVector("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getUnsignedVector("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getFloatVector("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getDoubleVector("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getStringVector("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getInt32("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getInt64("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getInt32Vector("missing"), eckit::Exception); EXPECT_THROWS_AS(conf.getInt64Vector("missing"), eckit::Exception); // Get with default values instead of throwing EXPECT(conf.getBool("missing", value_bool) == value_bool); EXPECT(conf.getInt("missing", value_int) == value_int); EXPECT(conf.getLong("missing", value_long) == value_long); EXPECT(conf.getUnsigned("missing", value_size_t) == value_size_t); EXPECT(conf.getFloat("missing", value_float) == value_float); EXPECT(conf.getDouble("missing", value_double) == value_double); EXPECT(conf.getString("missing", value_string) == value_string); EXPECT(conf.getIntVector("missing", value_arr_int) == value_arr_int); EXPECT(conf.getLongVector("missing", value_arr_long) == value_arr_long); EXPECT(conf.getUnsignedVector("missing", value_arr_size_t) == value_arr_size_t); EXPECT(conf.getFloatVector("missing", value_arr_float) == value_arr_float); EXPECT(conf.getDoubleVector("missing", value_arr_double) == value_arr_double); EXPECT(conf.getStringVector("missing", value_arr_string) == value_arr_string); EXPECT(conf.getInt32("missing", value_int32) == value_int32); EXPECT(conf.getInt64("missing", value_int64) == value_int64); EXPECT(conf.getInt32Vector("missing", value_arr_int32) == value_arr_int32); EXPECT(conf.getInt64Vector("missing", value_arr_int64) == value_arr_int64); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_yaml_empty") { PathName yamlpath = "test_yaml_empty.yaml"; std::string yamlstr( "office:\n" " manager :\n"); { std::ofstream yamlfile(yamlpath.localPath()); yamlfile << yamlstr; } LocalConfiguration emptyconf; // LocalConfiguration is empty EXPECT(emptyconf.empty()); YAMLConfiguration conf(yamlpath); LocalConfiguration office(conf, "office"); // LocalConfiguration has content EXPECT(!office.empty()); // Not sure this is correct yaml but since the parser accepts it we need to handle it LocalConfiguration nilconf(office, "manager"); // LocalConfiguration is nil EXPECT(nilconf.empty()); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_json_configuration") { PathName jsonpath = "test_json_configuration.json"; std::string jsonstr( "\n" "{" "\n" " \"manager\" : { \"name\" : \"Sidonia\" , \"office\" : 1 }," "\n" " \"staff\" : [" "\n" " { \"name\" : \"Suske\" , \"office\" : 2 }," "\n" " { \"name\" : \"Wiske\" , \"office\" : 3 }" "\n" " ]" "\n" "}" "\n"); { std::ofstream jsonfile(jsonpath.localPath()); jsonfile << jsonstr; } YAMLConfiguration conf_from_str(jsonstr); YAMLConfiguration conf(jsonpath); LocalConfiguration manager; std::vector staff; EXPECT(conf.get("manager", manager)); EXPECT(conf.get("staff", staff)); std::string name; int office; EXPECT(manager.get("name", name)); EXPECT(manager.get("office", office)); EXPECT(name == std::string("Sidonia")); EXPECT(office == 1); EXPECT(staff[0].get("name", name)); EXPECT(staff[0].get("office", office)); EXPECT(name == std::string("Suske")); EXPECT(office == 2); EXPECT(staff[1].get("name", name)); EXPECT(staff[1].get("office", office)); EXPECT(name == std::string("Wiske")); EXPECT(office == 3); } //---------------------------------------------------------------------------------------------------------------------- CASE("YAML configuration converts numbers to strings or numbers") { const char* text = R"YAML( --- base: one : "1" two : "2" three : 3 four : 4 neg7 : "-7" )YAML"; std::string cfgtxt(text); YAMLConfiguration conf(cfgtxt); LocalConfiguration local; EXPECT_NO_THROW(conf.get("base", local)); std::cerr << Colour::green << local << Colour::reset << std::endl; EXPECT_EQUAL(local.getString("one"), "1"); EXPECT_EQUAL(local.getString("two"), "2"); EXPECT_EQUAL(local.getString("three"), "3"); EXPECT_EQUAL(local.getString("four"), "4"); EXPECT_EQUAL(local.getInt("one"), 1); EXPECT_EQUAL(local.getLong("two"), 2); EXPECT_EQUAL(local.getInt32("three"), 3); EXPECT_EQUAL(local.getInt64("four"), 4); EXPECT_EQUAL(local.getString("neg7"), "-7"); EXPECT_EQUAL(local.getInt("neg7"), -7); } //---------------------------------------------------------------------------------------------------------------------- CASE("YAML configuration with null value") { const char* text = R"YAML( --- base: nothing : null )YAML"; std::string cfgtxt(text); YAMLConfiguration conf(cfgtxt); LocalConfiguration local; EXPECT_NO_THROW(conf.get("base", local)); std::cerr << Colour::green << local << Colour::reset << std::endl; EXPECT(local.isNull("nothing")); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_local_configuration") { LocalConfiguration local; { LocalConfiguration manager; manager.set("name", "Sidonia"); manager.set("office", 1); manager.set("height", 1.78); manager.set("free", false); std::vector staff(2); staff[0].set("name", "Suske"); staff[1].set("name", "Wiske"); staff[0].set("office", 2); staff[1].set("office", 3); local.set("manager", manager); local.set("staff", staff); local.set("books.count", 10); } const Configuration& conf = local; LocalConfiguration manager; std::vector staff; EXPECT(conf.get("manager", manager)); EXPECT(conf.isSubConfigurationList("staff")); EXPECT(conf.isConvertible("staff", staff)); EXPECT(conf.get("staff", staff)); std::string name; int office; EXPECT(manager.get("name", name)); EXPECT(manager.get("office", office)); EXPECT(name == std::string("Sidonia")); EXPECT(office == 1); EXPECT(staff[0].get("name", name)); EXPECT(staff[0].get("office", office)); EXPECT(name == std::string("Suske")); EXPECT(office == 2); EXPECT(staff[1].get("name", name)); EXPECT(staff[1].get("office", office)); EXPECT(name == std::string("Wiske")); EXPECT(office == 3); int books_count; EXPECT(conf.has("books")); EXPECT(conf.get("books.count", books_count)); EXPECT(books_count == 10); LocalConfiguration books; EXPECT(conf.isSubConfiguration("books")); conf.get("books", books); EXPECT(books.getInt("count") == 10); EXPECT(conf.isConvertible("books")); EXPECT(conf.isList("staff")); EXPECT(conf.isIntegral("manager.office")); EXPECT(!conf.isFloatingPoint("manager.office")); EXPECT(conf.isConvertible("manager.office")); EXPECT(conf.isConvertible("manager.office")); EXPECT(conf.isConvertible("manager.office")); EXPECT(conf.isConvertible("manager.office")); EXPECT(conf.isConvertible("manager.office")); EXPECT(conf.isConvertible("manager.office")); EXPECT(!conf.isConvertible("manager.office")); EXPECT(!conf.isConvertible("manager.office")); EXPECT(!conf.isConvertible("manager.office")); EXPECT(conf.isConvertible("manager.height")); EXPECT(conf.isConvertible("manager.height")); EXPECT(!conf.isConvertible("manager.height")); EXPECT(!conf.isConvertible("manager.height")); EXPECT(!conf.isConvertible("manager.height")); EXPECT(!conf.isConvertible("manager.height")); double manager_height; EXPECT(conf.get("manager.height", manager_height)); EXPECT(manager_height == 1.78); local.set("a", "a"); const eckit::Parametrisation& p = conf; EXPECT(!p.has("a.b")); } CASE("Hash a configuration") { std::unique_ptr h(eckit::HashFactory::instance().build("MD5")); LocalConfiguration cfg; cfg.set("name", "Sidonia"); cfg.set("office", 1); cfg.hash(*h); // std::cout << "MD5 " << h->digest() << std::endl; EXPECT(h->digest() == "9f060b35735e98b0fdc0bf4c2d6d6d8d"); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/config/test_resource.cc0000664000175000017500000000605615161702250020615 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/config/ResourceMgr.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/runtime/Tool.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" #include "eckit/types/Types.h" using namespace std; using namespace eckit; using namespace eckit::testing; using namespace eckit::types; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_default") { std::string s = Resource("s", "some"); EXPECT(s == "some"); double d = Resource("d", 777.7); EXPECT(is_approximately_equal(d, 777.7, 0.0001)); // accept 0.0001% tolerance } //---------------------------------------------------------------------------------------------------------------------- CASE("test_vector_long") { std::vector def(3, 77); std::vector v = Resource >("listlong;-listlong", def); EXPECT(v[0] == 88); EXPECT(v[1] == 99); EXPECT(v[2] == 11); EXPECT(v[3] == 22); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_command_line") { int myint = Resource("integer;-integer", 0); EXPECT(myint == 100); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_environment_var") { char v[] = "TEST_ENV_INT=333"; putenv(v); int intenv = Resource("intEnv;$TEST_ENV_INT", 777); EXPECT(intenv == 333); char foo[] = "FOO=1Mb"; putenv(foo); long l1 = Resource("$FOO", 0); EXPECT(l1 == 1024 * 1024); long l2 = Resource("$FOO;-foo", 0); EXPECT(l2 == 1024 * 1024); long l3 = Resource("-foo;$FOO", 0); EXPECT(l3 == 1024 * 1024); long l4 = Resource("$FOO;foo;-foo", 0); EXPECT(l4 == 1024 * 1024); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_libresource") { long i = LibResource("foo-bar;$FOOBAR", 1024 * 1024); EXPECT(i == 1024 * 1024); char v[] = "TEST_FOOBAZ=fooBAZ"; putenv(v); std::string foobaz = LibResource("foo-baz;$TEST_FOOBAZ", "foobazzzzzzz"); EXPECT(foobaz == "fooBAZ"); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/config/CMakeLists.txt0000664000175000017500000000041615161702250020152 0ustar alastairalastairecbuild_add_test( TARGET eckit_test_config_resource SOURCES test_resource.cc ARGS -integer 100 -listlong 88,99,11,22 LIBS eckit ) ecbuild_add_test( TARGET eckit_test_config_configuration SOURCES test_configuration.cc LIBS eckit ) eckit-2.0.7/tests/geometry/0000775000175000017500000000000015161702250015777 5ustar alastairalastaireckit-2.0.7/tests/geometry/test_coordinate_helpers.cc0000664000175000017500000001416315161702250023223 0ustar alastairalastair/* * (C) Copyright 2023 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ #include #include "eckit/geometry/CoordinateHelpers.h" #include "eckit/geometry/Point2.h" #include "eckit/testing/Test.h" namespace eckit::test { using namespace geometry; // ----------------------------------------------------------------------------- CASE("normalise angles") { EXPECT(0. == normalise_angle(360., 0.)); EXPECT(14. == normalise_angle(374., 0.)); EXPECT(14. == normalise_angle(374., -90.)); EXPECT(374. == normalise_angle(374., 90.)); EXPECT(14. == normalise_angle(-346., 0.)); EXPECT(14. == normalise_angle(-346., -90.)); EXPECT(374. == normalise_angle(-346., 90.)); EXPECT(0. == normalise_angle(360. * 1e12, 0.)); EXPECT(14. == normalise_angle(360. * 1e12 + 14, 0.)); EXPECT(0. == normalise_angle(-360. * 1e12, 0.)); EXPECT(14. == normalise_angle(-360. * 1e12 + 14, 0.)); } CASE("normalise angles that should stay bit-identical") { std::vector lats{ 0x1.a99999999999fp+3, // 13.3 0x1.7599999999999p+5, // 46.7 -0x1.37823af2187f7p+4, // -19.4692944962371 0x1.14f26c8adc252p+3, // 8.65459277268488 0x1.237e9f537dd2dp+5, // 36.4368273279928 0x1.eb74b977e1e89p+5, // 61.431994377691 0x1.1008717af4f67p+6, // 68.0082453929914 -0x1.b4f88656270d9p+4, // -27.3106749883017 -0x1.eb22f87f6ac12p+1, // -3.83700472089323 0x1.40de11e0c3e99p+4, // 20.0542162685293 0x1.4aeba99be1331p+5, // 41.3650695970631 0x1.aa5c50f727ae6p+5, // 53.2950763043389 -0x1.556ccf04ef1bbp+4, // -21.3390646164641 0x1.556ccf04ef1bbp+4, // 21.3390646164641 0x1.388f683df92bbp+5, // 39.070023044745 -0x1.40de11e0c3e9dp+4, // -20.0542162685293 0x1.eb22f87f6abf5p+1, // 3.83700472089321 0x1.b4f88656270d7p+4, // 27.3106749883017 }; for (const auto& lat : lats) { double normalised = normalise_angle(lat, -90.); // std::cout << std::hexfloat << lat << "\t : " << std::defaultfloat << std::setprecision(17) << lat << // std::endl; std::cout << std::hexfloat << normalised << "\t : " << std::defaultfloat << std::setprecision(17) // << normalised << std::endl; EXPECT_EQUAL(normalised, lat); } std::vector lons{ -0x1.3f0f4411db559p+5, // -39.8824540515004 -0x1.63664f7d2181dp+5, // -44.4249563003398 -0x1.75e470fd085aap+5, // -46.7365436332869 -0x1.b2a6314996231p+4, // -27.1655743479225 -0x1.f720e2a9525edp+5, // -62.8910573223364 -0x1.236723c039272p+5, // -36.425361158127 -0x1.7f9f1a40a5d1fp+4, // -23.9763433957388 0x1.ffffffffffffep+0, // 2.0 0x1.0b907154a92f7p+6, // 66.8910573223364 0x1.436723c039272p+5, // 40.425361158127 0x1.bf9f1a40a5d1fp+4, // 27.9763433957388 0x1.0f266c20b79f9p+7, // 135.57504369966 0x1.787bbbb54c676p+6, // 94.1208332374461 0x1.95e470fd085aap+5, // 50.7365436332869 0x1.1bd0dd7b42b69p+7, // 141.907939769643 0x1.19981bd70b549p+6, // 70.3985437012353 0x1.50bc8a12f525bp+5, // 42.0920602303565 0x1.cb2a2664f7bbdp+6, // 114.791162088032 0x1.6784444ab398ap+6, // 89.8791667625539 0x1.83664f7d2181ep+5, // 48.4249563003398 0x1.380c1cb7eb45dp+7, // 156.023656604261 0x1.d46f8eab56d0bp+6, // 117.108942677664 0x1.5f0f4411db559p+5, // 43.8824540515004 }; for (const auto& lon : lons) { double normalised = normalise_angle(lon, -180.); // std::cout << std::hexfloat << lon << "\t : " << std::defaultfloat << std::setprecision(17) << lon << // std::endl; std::cout << std::hexfloat << normalised << "\t : " << std::defaultfloat << std::setprecision(17) // << normalised << std::endl; EXPECT_EQUAL(normalised, lon); } } CASE("normalise angles that should be reproducible") { constexpr double minimum = -90.; std::vector> reproduce{ // angle normalised {0x1.a99999999999fp+3, 0x1.a99999999999fp+3}, {0x1.a99999999999fp+3 - 360., 0x1.a9999999999ap+3}, {0x1.a99999999999fp+3 - 720., 0x1.a99999999998p+3}, {0x1.a99999999999fp+3 + 360., 0x1.a9999999999ap+3}, {0x1.a99999999999fp+3 + 720., 0x1.a99999999998p+3}, }; for (const auto& [angle, normalised_ref] : reproduce) { double normalised = normalise_angle(angle, minimum); // std::cout << std::hexfloat << angle << "\t : " << std::defaultfloat << std::setprecision(17) << angle << // std::endl; std::cout << std::hexfloat << normalised << "\t : " << std::defaultfloat << std::setprecision(17) // << normalised << std::endl; EXPECT(normalised == normalised_ref); } } CASE("canonicalise on sphere") { using types::is_approximately_equal; const Point2 p1(108., 32.); // Worse coordinates for the same point: const Point2 p2(-252., 32.); const Point2 p3(288., 148.); const Point2 p4(108., -328.); // Check each of these is correctly shifted back to original point: const Point2 q2 = canonicaliseOnSphere(p2); const Point2 q3 = canonicaliseOnSphere(p3); const Point2 q4 = canonicaliseOnSphere(p4); EXPECT(p1.x() == q2.x()); EXPECT(p1.y() == q2.y()); EXPECT(p1.x() == q3.x()); EXPECT(p1.y() == q3.y()); EXPECT(p1.x() == q4.x()); EXPECT(p1.y() == q4.y()); // Check with longitude offset const Point2 q2b = canonicaliseOnSphere(p2, -90.); EXPECT(q2b.x() == 108.); EXPECT(q2b.y() == 32.); const Point2 q2c = canonicaliseOnSphere(p2, 90.); EXPECT(q2c.x() == 108.); EXPECT(q2c.y() == 32.); const Point2 q2d = canonicaliseOnSphere(p2, 180.); EXPECT(q2d.x() == 468.); EXPECT(q2d.y() == 32.); } // ----------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geometry/test_kdtree.cc0000664000175000017500000003002315161702250020621 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/container/KDTree.h" #include "eckit/geometry/Point2.h" #include "eckit/os/Semaphore.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; using namespace eckit::geometry; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- struct TestTreeTrait { using Point = Point2; using Payload = double; }; //---------------------------------------------------------------------------------------------------------------------- /// \brief Class used to test whether any point in a kd-tree lies in the interior of an /// axis-aligned box. template class PointInBoxInteriorFinder { public: using KDTree = eckit::KDTreeX; using Point = typename KDTree::Point; private: using Alloc = typename KDTree::Alloc; using Node = typename KDTree::Node; public: /// \brief Returns true if any point in \p tree lies in the interior of the specified /// axis-aligned box. /// /// \param tree /// Tree to search. /// \param lbound /// Lower-left corner of the axis-aligned box. /// \param ubound /// Upper-right corner of the axis-aligned box. static bool isAnyPointInBoxInterior(const KDTree& tree, const Point& lbound, const Point& ubound) { if (!tree.root_) { return false; } Alloc& alloc = tree.alloc_; Node* root = alloc.convert(tree.root_, static_cast(nullptr)); return isAnyPointInBoxInterior(root, alloc, lbound, ubound); } private: /// \brief Returns true if the point stored in \p node or any of its descendants lies in the /// interior of the axis-aligned box with bottom-left and top-right corners at /// \p lbound and \p ubound. static bool isAnyPointInBoxInterior(const Node* node, Alloc& alloc, const Point& lbound, const Point& ubound) { const Point& point = node->value().point(); if (isPointInBoxInterior(point, lbound, ubound)) { return true; } const size_t axis = node->axis(); if (lbound.x(axis) < point.x(axis)) { if (Node* left = node->left(alloc)) { if (isAnyPointInBoxInterior(left, alloc, lbound, ubound)) { return true; } } } if (ubound.x(axis) > point.x(axis)) { if (Node* right = node->right(alloc)) { if (isAnyPointInBoxInterior(right, alloc, lbound, ubound)) { return true; } } } return false; } /// \brief Returns true if \p point is in the interior of the axis-aligned box /// with bottom-left and top-right corners at \p lbound and \p ubound. static bool isPointInBoxInterior(const Point& point, const Point& lbound, const Point& ubound) { for (size_t d = 0; d < Point::DIMS; ++d) { if (point.x(d) <= lbound.x(d) || point.x(d) >= ubound.x(d)) { return false; } } return true; } }; //---------------------------------------------------------------------------------------------------------------------- /// \brief Returns true if any point in \p tree is in the interior of the axis-aligned box /// with bottom-left and top-right corners at \p lbound and \p ubound. template bool isAnyPointInBoxInterior(const eckit::KDTreeX& tree, const typename eckit::KDTreeX::Point& lbound, const typename eckit::KDTreeX::Point& ubound) { return PointInBoxInteriorFinder::isAnyPointInBoxInterior(tree, lbound, ubound); } //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_container_kdtree_constructor") { using Tree = KDTreeMemory; Tree kd; using Point = Tree::PointType; std::vector points; // test it for single closest point for (unsigned int i = 0; i < 10; i++) { for (unsigned int j = 0; j < 10; j++) { Point p = Point(double(i), double(j)); Tree::Value v(p, 99.9); points.push_back(v); } } kd.build(points.begin(), points.end()); // pick some point from the vector Point refPoint = points[points.size() / 2].point(); // perturb it a little Point delta(0.1, 0.1); Point testPoint = Point::add(refPoint, delta); Log::info() << "testPoint perturb " << testPoint.x(0) << ", " << testPoint.x(1) << std::endl; Point nr = kd.nearestNeighbour(testPoint).point(); // we should find the same point for (unsigned int i = 0; i < Point::dimensions(); i++) { EXPECT(nr.x(i) == refPoint.x(i)); } // test exact match to a point nr = kd.nearestNeighbour(refPoint).point(); for (unsigned int i = 0; i < Point::dimensions(); i++) { EXPECT(nr.x(i) == refPoint.x(i)); } // test "off the scale" - i.e. not within a group of points delta = Point(1000.0, 0.0); // add it to the last point testPoint = Point::add(points.back().point(), delta); nr = kd.nearestNeighbour(testPoint).point(); for (unsigned int i = 0; i < Point::dimensions(); i++) { EXPECT(nr.x(i) == points.back().point().x(i)); } // and negatively // delta = Point(-1000.0, 0.0); // add it to the point() point testPoint = Point::add(points.front().point(), delta); nr = kd.nearestNeighbour(testPoint).point(); for (unsigned int i = 0; i < Point::dimensions(); i++) { EXPECT(nr.x(i) == points.front().point().x(i)); } // test N nearest refPoint = points[points.size() / 2].point(); // move this point so it lies between four equally delta = Point(0.5, 0.5); testPoint = Point::add(refPoint, delta); Tree::NodeList nn = kd.kNearestNeighbours(testPoint, 4); for (Tree::NodeList::iterator it = nn.begin(); it != nn.end(); ++it) { Point diff = Point::sub(it->point(), testPoint); // make sure we differ by 0.5 along each axis for (unsigned int i = 0; i < Point::dimensions(); ++i) { Log::info() << "distance along point " << Point::distance(Point(0.0, 0.0), diff, i) << std::endl; EXPECT(Point::distance(Point(0.0, 0.0), diff, i) == 0.5); } } // Test a custom visitor. The purpose of doing that in this test is to ensure that the public // interface of KDTree is sufficient to write a custom class traversing the tree. delta = Point(0.25, 0.25); Point lbound = Point::sub(refPoint, delta); Point ubound = Point::add(refPoint, delta); EXPECT(isAnyPointInBoxInterior(kd, lbound, ubound)); delta = Point(0.5, 0.5); lbound = Point::add(lbound, delta); ubound = Point::add(ubound, delta); EXPECT_NOT(isAnyPointInBoxInterior(kd, lbound, ubound)); // Test size() EXPECT_EQUAL(kd.size(), points.size()); } CASE("test_eckit_container_kdtree_insert") { using Tree = KDTreeMemory; Tree kd; using Point = Tree::PointType; std::vector points; // test it for single closest point for (unsigned int i = 0; i < 10; i++) { for (unsigned int j = 0; j < 10; j++) { Point p = Point(double(i), double(j)); Tree::Value v(p, 99.9); points.push_back(v); kd.insert(v); } } // pick some point from the vector Point refPoint = points[points.size() / 2].point(); // perturb it a little Point delta(0.1, 0.1); Point testPoint = Point::add(refPoint, delta); Log::info() << "testPoint perturb " << testPoint.x(0) << ", " << testPoint.x(1) << std::endl; Point nr = kd.nearestNeighbour(testPoint).point(); // we should find the same point for (unsigned int i = 0; i < Point::dimensions(); i++) { EXPECT(nr.x(i) == refPoint.x(i)); } // test exact match to a point nr = kd.nearestNeighbour(refPoint).point(); for (unsigned int i = 0; i < Point::dimensions(); i++) { EXPECT(nr.x(i) == refPoint.x(i)); } // test "off the scale" - i.e. not within a group of points delta = Point(1000.0, 0.0); // add it to the last point testPoint = Point::add(points.back().point(), delta); nr = kd.nearestNeighbour(testPoint).point(); for (unsigned int i = 0; i < Point::dimensions(); i++) { EXPECT(nr.x(i) == points.back().point().x(i)); } // and negatively // delta = Point(-1000.0, 0.0); // add it to the point() point testPoint = Point::add(points.front().point(), delta); nr = kd.nearestNeighbour(testPoint).point(); for (unsigned int i = 0; i < Point::dimensions(); i++) { EXPECT(nr.x(i) == points.front().point().x(i)); } // test N nearest refPoint = points[points.size() / 2].point(); // move this point so it lies between four equally delta = Point(0.5, 0.5); testPoint = Point::add(refPoint, delta); Tree::NodeList nn = kd.kNearestNeighbours(testPoint, 4); for (Tree::NodeList::iterator it = nn.begin(); it != nn.end(); ++it) { Point diff = Point::sub(it->point(), testPoint); // make sure we differ by 0.5 along each axis for (unsigned int i = 0; i < Point::dimensions(); ++i) { Log::info() << "distance along point " << Point::distance(Point(0.0, 0.0), diff, i) << std::endl; EXPECT(Point::distance(Point(0.0, 0.0), diff, i) == 0.5); } } } CASE("test_kdtree_mapped") { using Tree = KDTreeMapped; using Point = Tree::PointType; std::vector points; for (unsigned int i = 0; i < 10; i++) { for (unsigned int j = 0; j < 10; j++) { Point p = Point(double(i), double(j)); Tree::Value v(p, 99.9); points.push_back(v); } } auto passTest = [&](Tree& kd) -> bool { // pick some point from the vector Point refPoint = points[points.size() / 2].point(); // perturb it a little Point delta(0.1, 0.1); Point testPoint = Point::add(refPoint, delta); Point nr = kd.nearestNeighbour(testPoint).point(); // we should find the same point for (unsigned int i = 0; i < Point::dimensions(); i++) { if (nr.x(i) != refPoint.x(i)) { return false; } } return true; }; eckit::PathName path("test_kdtree_mapped.kdtree"); // Write file with kdtree { if (path.exists()) { path.unlink(); } Tree kd(path, points.size(), 0); EXPECT_EQUAL(kd.size(), 0); kd.build(points); EXPECT_EQUAL(kd.size(), points.size()); EXPECT(passTest(kd)); } // Load file with kdtree { Tree kd(path, 0, 0); // Cannot insert point as the tree is readonly EXPECT_THROWS_AS(kd.insert(points.front()), eckit::AssertionFailed); // Cannot build with points as the tree is readonly EXPECT_THROWS_AS(kd.build(points), eckit::AssertionFailed); EXPECT_EQUAL(kd.size(), points.size()); EXPECT(passTest(kd)); } } CASE("test_kdtree_iterate_empty") { using Tree = KDTreeMemory; Tree kd; size_t count{0}; for (auto& item : kd) { count++; } EXPECT_EQUAL(count, 0); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/geometry/test_sphere.cc0000664000175000017500000002363615161702250020645 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/geometry/Point2.h" #include "eckit/geometry/Point3.h" #include "eckit/geometry/SphereT.h" #include "eckit/geometry/UnitSphere.h" #include "eckit/testing/Test.h" namespace eckit::test { using namespace geometry; struct PointLonLat : Point2 { PointLonLat(double x, double y) : Point2(x, y) {} const double& lon() const { return x_[0]; } const double& lat() const { return x_[1]; } }; struct PointXYZ : Point3 { const double& x() const { return x_[0]; } const double& y() const { return x_[1]; } const double& z() const { return x_[2]; } }; // set sphere struct DatumTwoUnits { static double radius() { return 2.; } }; using TwoUnitsSphere = SphereT; const double R = UnitSphere::radius(); // ----------------------------------------------------------------------------- // test unit sphere radius CASE("test unit sphere radius") { EXPECT(UnitSphere::radius() == 1.); } // ----------------------------------------------------------------------------- // test unit sphere poles CASE("test unit sphere north pole") { const PointLonLat ll1(0., 90.); PointXYZ p; UnitSphere::convertSphericalToCartesian(ll1, p); EXPECT(p.x() == 0); EXPECT(p.y() == 0); EXPECT(p.z() == R); } CASE("test unit sphere south pole") { const PointLonLat ll1(0., -90.); PointXYZ p; UnitSphere::convertSphericalToCartesian(ll1, p); EXPECT(p.x() == 0); EXPECT(p.y() == 0); EXPECT(p.z() == -R); } // ----------------------------------------------------------------------------- // test unit sphere quadrants CASE("test unit sphere lon 0") { const PointLonLat ll1(0., 0.); const PointLonLat ll2(-360., 0.); PointXYZ p, q; UnitSphere::convertSphericalToCartesian(ll1, p); UnitSphere::convertSphericalToCartesian(ll2, q); EXPECT(p.x() == R); EXPECT(p.y() == 0); EXPECT(p.z() == 0); EXPECT(PointXYZ::equal(p, q)); } CASE("test unit sphere lon 90") { const PointLonLat ll1(90., 0.); const PointLonLat ll2(-270., 0.); PointXYZ p, q; UnitSphere::convertSphericalToCartesian(ll1, p); UnitSphere::convertSphericalToCartesian(ll2, q); EXPECT(p.x() == 0); EXPECT(p.y() == R); EXPECT(p.z() == 0); EXPECT(PointXYZ::equal(p, q)); } CASE("test unit sphere lon 180") { const PointLonLat ll1(180., 0.); const PointLonLat ll2(-180., 0.); PointXYZ p, q; UnitSphere::convertSphericalToCartesian(ll1, p); UnitSphere::convertSphericalToCartesian(ll2, q); EXPECT(p.x() == -R); EXPECT(p.y() == 0); EXPECT(p.z() == 0); EXPECT(PointXYZ::equal(p, q)); } CASE("test unit sphere lon 270") { const PointLonLat ll1(270., 0.); const PointLonLat ll2(-90., 0.); PointXYZ p, q; UnitSphere::convertSphericalToCartesian(ll1, p); UnitSphere::convertSphericalToCartesian(ll2, q); EXPECT(p.x() == 0); EXPECT(p.y() == -R); EXPECT(p.z() == 0); EXPECT(PointXYZ::equal(p, q)); } // ----------------------------------------------------------------------------- // test unit sphere octants const double L = R * std::sqrt(2) / 2.; CASE("test unit sphere lon 45") { const PointLonLat ll1(45., 0.); const PointLonLat ll2(-315., 0.); PointXYZ p, q; UnitSphere::convertSphericalToCartesian(ll1, p); UnitSphere::convertSphericalToCartesian(ll2, q); EXPECT(eckit::types::is_approximately_equal(p.x(), L)); EXPECT(eckit::types::is_approximately_equal(p.y(), L)); EXPECT(p.z() == 0); EXPECT(PointXYZ::equal(p, q)); } CASE("test unit sphere lon 135") { const PointLonLat ll1(135., 0.); const PointLonLat ll2(-225., 0.); PointXYZ p, q; UnitSphere::convertSphericalToCartesian(ll1, p); UnitSphere::convertSphericalToCartesian(ll2, q); EXPECT(eckit::types::is_approximately_equal(p.x(), -L)); EXPECT(eckit::types::is_approximately_equal(p.y(), L)); EXPECT(p.z() == 0); EXPECT(PointXYZ::equal(p, q)); } CASE("test unit sphere lon 225") { const PointLonLat ll1(225., 0.); const PointLonLat ll2(-135., 0.); PointXYZ p, q; UnitSphere::convertSphericalToCartesian(ll1, p); UnitSphere::convertSphericalToCartesian(ll2, q); EXPECT(eckit::types::is_approximately_equal(p.x(), -L)); EXPECT(eckit::types::is_approximately_equal(p.y(), -L)); EXPECT(p.z() == 0); EXPECT(PointXYZ::equal(p, q)); } CASE("test unit sphere lon 315") { const PointLonLat ll1(315., 0.); const PointLonLat ll2(-45., 0.); PointXYZ p, q; UnitSphere::convertSphericalToCartesian(ll1, p); UnitSphere::convertSphericalToCartesian(ll2, q); EXPECT(eckit::types::is_approximately_equal(p.x(), L)); EXPECT(eckit::types::is_approximately_equal(p.y(), -L)); EXPECT(p.z() == 0); EXPECT(PointXYZ::equal(p, q)); } // ----------------------------------------------------------------------------- // test unit sphere with non-canonical latitudes outside [-90, 90] CASE("test unit sphere lat 100") { const PointLonLat ll1(0., 100.); const PointLonLat ll2(180., 80.); PointXYZ p, q; // Default behavior throws EXPECT_THROWS_AS(UnitSphere::convertSphericalToCartesian(ll1, p), eckit::BadValue); UnitSphere::convertSphericalToCartesian(ll1, p, 0., true); UnitSphere::convertSphericalToCartesian(ll2, q, 0., true); // sin(x) and sin(pi-x) are not bitwise identical EXPECT(eckit::types::is_approximately_equal(p.x(), q.x())); EXPECT(eckit::types::is_approximately_equal(p.y(), q.y())); EXPECT(eckit::types::is_approximately_equal(p.z(), q.z())); } CASE("test unit sphere lat 290") { const PointLonLat ll1(15., 290.); const PointLonLat ll2(15., -70.); PointXYZ p, q; // Default behavior throws EXPECT_THROWS_AS(UnitSphere::convertSphericalToCartesian(ll1, p), eckit::BadValue); UnitSphere::convertSphericalToCartesian(ll1, p, 0., true); UnitSphere::convertSphericalToCartesian(ll2, q, 0., true); // sin(x) and sin(pi-x) are not bitwise identical EXPECT(eckit::types::is_approximately_equal(p.x(), q.x())); EXPECT(eckit::types::is_approximately_equal(p.y(), q.y())); EXPECT(eckit::types::is_approximately_equal(p.z(), q.z())); } CASE("test unit sphere lat -120") { const PointLonLat ll1(45., -120.); const PointLonLat ll2(225., -60.); PointXYZ p, q; // Default behavior throws EXPECT_THROWS_AS(UnitSphere::convertSphericalToCartesian(ll1, p), eckit::BadValue); UnitSphere::convertSphericalToCartesian(ll1, p, 0., true); UnitSphere::convertSphericalToCartesian(ll2, q, 0., true); // sin(x) and sin(pi-x) are not bitwise identical EXPECT(eckit::types::is_approximately_equal(p.x(), q.x())); EXPECT(eckit::types::is_approximately_equal(p.y(), q.y())); EXPECT(eckit::types::is_approximately_equal(p.z(), q.z())); } // ----------------------------------------------------------------------------- // test unit sphere distances with non-canonical coordinates CASE("test unit sphere distances") { const PointLonLat P1(-71.6, -33.); // Valparaíso const PointLonLat P2(121.8, 31.4); // Shanghai // Same points with added shifts const PointLonLat P1b(288.4, -33.); // Valparaíso + longitude shift const PointLonLat P2b(301.8, 148.6); // Shanghai + latitude/longitude shift const PointLonLat P2c(-58.2, -211.4); // Shanghai + latitude/longitude shift const double d0 = UnitSphere::distance(P1, P2); const double d1 = UnitSphere::distance(P1b, P2); const double d2 = UnitSphere::distance(P1, P2b); const double d3 = UnitSphere::distance(P1, P2c); EXPECT(eckit::types::is_approximately_equal(d0, d1)); EXPECT(eckit::types::is_approximately_equal(d0, d2)); EXPECT(eckit::types::is_approximately_equal(d0, d3)); } // ----------------------------------------------------------------------------- // test unit sphere area CASE("test unit sphere area globe") { EXPECT(UnitSphere::area() == 4. * M_PI * R * R); } CASE("test unit sphere area hemispheres") { const PointLonLat ll1(-180., 90.); const PointLonLat ll2(180., 0.); const PointLonLat ll3(-180., 0.); const PointLonLat ll4(180., -90.); const double area_hemisphere_north = UnitSphere::area(ll1, ll2); const double area_hemisphere_south = UnitSphere::area(ll3, ll4); EXPECT(area_hemisphere_north == 0.5 * UnitSphere::area()); EXPECT(area_hemisphere_north == area_hemisphere_south); } // ----------------------------------------------------------------------------- // test two units sphere CASE("test two units sphere radius") { EXPECT(TwoUnitsSphere::radius() == 2.); } CASE("test two units sphere distances") { const PointLonLat P1(-71.6, -33.); // Valparaíso const PointLonLat P2(121.8, 31.4); // Shanghai const double d_sphere_1 = UnitSphere::distance(P1, P2); const double d_sphere_2 = TwoUnitsSphere::distance(P1, P2); EXPECT(2. * d_sphere_1 == d_sphere_2); } CASE("test two units sphere areas") { const double area_sphere_1 = UnitSphere::area(); const double area_sphere_2 = TwoUnitsSphere::area(); EXPECT(4. * area_sphere_1 == area_sphere_2); } CASE("test two units sphere sub areas") { const PointLonLat P1(-71.6, -33.); // Valparaíso const PointLonLat P2(121.8, 31.4); // Shanghai const double sub_area_sphere_1 = UnitSphere::area(P2, P1); const double sub_area_sphere_2 = TwoUnitsSphere::area(P2, P1); EXPECT(4. * sub_area_sphere_1 == sub_area_sphere_2); } // ----------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geometry/test_great_circle.cc0000664000175000017500000002020115161702250021763 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/geometry/GreatCircle.h" #include "eckit/geometry/Point2.h" #include "eckit/testing/Test.h" namespace eckit::test { using namespace geometry; struct PointLonLat : Point2 { PointLonLat(double x, double y) : Point2(x, y) {} const double& lon() const { return x_[0]; } const double& lat() const { return x_[1]; } }; // ----------------------------------------------------------------------------- // test great circles CASE("test great circles intersections") { using types::is_approximately_equal; using types::is_approximately_greater_or_equal; auto is_approximately_equal_longitude = [](double lon1, double lon2, double epsilon = std::numeric_limits::epsilon()) -> bool { while (lon2 < lon1) { lon2 += 360; } while (lon1 >= lon1 + 360) { lon2 -= 360; } return is_approximately_equal(lon1, lon2, epsilon) || is_approximately_equal(lon1, lon2 - 360, epsilon); }; auto is_approximately_pole = [](double lat, double epsilon = std::numeric_limits::epsilon()) -> bool { return is_approximately_equal(std::abs(lat), 90., epsilon); }; auto is_approximately_equator = [](double lat, double epsilon = std::numeric_limits::epsilon()) -> bool { return is_approximately_equal(lat, 0., epsilon); }; const std::vector latitudes{ 90, 60, 45, 30, 0, -30, -45, -60, -90, }; const std::vector longitudes{ -181, -180, -135, -90, -45, 0, 45, 90, 135, 180, 225, 270, 315, 360, 361, }; const std::vector antipodes{ {0, 0}, {180, 0}, {-180, 0}, {0, 0}, {-90, 0}, {90, 0}, {90, 0}, {-90, 0}, {0, 90}, {0, -90}, {0, -90}, {0, 90}, {45, 45}, {225, -45}, }; SECTION("example intersection with meridian and parallel") { // latitude at Valparaíso-Shanghai mid-point const PointLonLat P1(-71.6, -33.); const PointLonLat P2(121.8, 31.4); GreatCircle gc(P1, P2); const PointLonLat mid(-159.18, -6.81); auto lats = gc.latitude(mid.lon()); EXPECT(lats.size() == 1 && is_approximately_equal(lats[0], mid.lat(), 0.01)); auto lons = gc.longitude(mid.lat()); EXPECT(lons.size() == 2); EXPECT(is_approximately_equal_longitude(lons[0], mid.lon(), 0.01) || is_approximately_equal_longitude(lons[1], mid.lon(), 0.01)); } SECTION("mal-formed great circle") { for (size_t i = 0; i < antipodes.size(); i += 2) { const PointLonLat& A(antipodes[i]); const PointLonLat& B(antipodes[i + 1]); EXPECT_THROWS_AS(GreatCircle(A, A), BadValue); EXPECT_THROWS_AS(GreatCircle(B, B), BadValue); EXPECT_THROWS_AS(GreatCircle(A, B), BadValue); if (is_approximately_pole(A.lat())) { for (double lon1_gc : longitudes) { for (double lon2_gc : longitudes) { EXPECT_THROWS_AS(GreatCircle({lon1_gc, A.lat()}, {lon2_gc, A.lat()}), BadValue); EXPECT_THROWS_AS(GreatCircle({lon1_gc, B.lat()}, {lon2_gc, B.lat()}), BadValue); } } } } } SECTION("intersection at quadrants") { for (double lat_gc : latitudes) { if (!is_approximately_pole(lat_gc) && !is_approximately_equator(lat_gc)) { for (double lon_gc : longitudes) { GreatCircle gc({lon_gc, lat_gc}, {lon_gc + 90, 0}); EXPECT(!gc.crossesPoles()); auto lon_at_equator = gc.longitude(0); EXPECT(lon_at_equator.size() == 2); EXPECT((is_approximately_equal_longitude(lon_gc + 90, lon_at_equator[0]) && is_approximately_equal_longitude(lon_gc - 90, lon_at_equator[1])) || (is_approximately_equal_longitude(lon_gc - 90, lon_at_equator[0]) && is_approximately_equal_longitude(lon_gc + 90, lon_at_equator[1]))); auto lon_extrema1 = gc.longitude(lat_gc); EXPECT(lon_extrema1.size() == 1 && is_approximately_equal_longitude(lon_extrema1[0], lon_gc, 0.01)); auto lon_extrema2 = gc.longitude(-lat_gc); EXPECT(lon_extrema2.size() == 1 && is_approximately_equal_longitude(lon_extrema2[0], lon_gc + 180, 0.01)); } } } } SECTION("intersection with parallels when crossing the poles") { for (double lon : longitudes) { for (double lat : latitudes) { { GreatCircle gc({lon, -10}, {lon, 10}); EXPECT(gc.crossesPoles()); auto lons = gc.longitude(lat); size_t N = is_approximately_pole(lat) ? 1 : 2; EXPECT(lons.size() == N); if (N == 1) { EXPECT(is_approximately_equal_longitude(lons[0], lon)); } else { EXPECT(is_approximately_equal_longitude(lons[0] + 180, lons[1])); EXPECT(is_approximately_equal_longitude(lons[0], lon) || is_approximately_equal_longitude(lons[1], lon)); } } if (!is_approximately_pole(lat) && !is_approximately_equator(lat)) { GreatCircle gc({lon, lat}, {lon + 180, lat}); EXPECT(gc.crossesPoles()); auto lons = gc.longitude(lat); EXPECT(lons.size() == 2); EXPECT(is_approximately_equal_longitude(lons[0] + 180, lons[1])); EXPECT(is_approximately_equal_longitude(lons[0], lon) || is_approximately_equal_longitude(lons[1], lon)); } } } } SECTION("intersection with parallels") { for (double lat_gc : latitudes) { if (/* avoid mal-forming */ !is_approximately_pole(lat_gc)) { for (double lat : latitudes) { GreatCircle gc({-1, lat_gc}, {1, lat_gc}); EXPECT(!gc.crossesPoles()); auto lons = gc.longitude(lat); size_t N = is_approximately_equator(lat_gc) ? 0. : is_approximately_greater_or_equal(std::abs(lat_gc), std::abs(lat)) ? 2 : 0; EXPECT(lons.size() == N); for (auto lon : lons) { auto lats = gc.latitude(lon); EXPECT(lats.size() == 1 && is_approximately_equal(lats[0], lat, 0.01)); } } } } } SECTION("equator great circle intersection with meridian and parallel") { for (double lon : longitudes) { GreatCircle eq({lon - 1, 0}, {lon + 1, 0}); EXPECT(!eq.crossesPoles()); // non-intersection with parallels for (double lat : latitudes) { EXPECT(eq.longitude(lat).empty()); } // intersect one latitude only, for specific longitudes auto lats = eq.latitude(lon); EXPECT(lats.size() == 1 && is_approximately_equator(lats[0])); } } } // ----------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geometry/test_points.cc0000664000175000017500000000643115161702250020665 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geometry/Point2.h" #include "eckit/geometry/Point3.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; using namespace eckit::geometry; namespace eckit::test { CASE("Inits to Zero") { Point2 q; EXPECT(q[XX] == 0.); EXPECT(q[YY] == 0.); } CASE("Inits to Array/Point") { Point2 q = {4.0, 5.0}; EXPECT(q[XX] == 4.0); EXPECT(q[YY] == 5.0); EXPECT(q[LON] == 4.0); EXPECT(q[LAT] == 5.0); Point2 r(q); EXPECT(r[XX] == 4.0); EXPECT(r[YY] == 5.0); Point3 p = {1.0, 2.0, 3.0}; Point3 s(p); EXPECT(s[XX] == 1.0); EXPECT(s[YY] == 2.0); EXPECT(s[ZZ] == 3.0); } CASE("Point2 addition") { Point2 p1 = {1.0, 2.0}; Point2 p2 = {2.0, 4.0}; Point2 r = p1 + p2; EXPECT(r[XX] == 3.0); EXPECT(r[YY] == 6.0); } CASE("Point2 subtraction") { Point2 p1 = {2.0, 5.0}; Point2 p2 = {1.0, 2.0}; Point2 r = p1 - p2; EXPECT(r[XX] == 1.0); EXPECT(r[YY] == 3.0); } CASE("Point2 scaling") { Point2 p1 = {1.0, 2.0}; Point2 p2(p1); Point2 r = p1 * 42.0; EXPECT(r[XX] == 42.0); EXPECT(r[YY] == 84.0); Point2 oo; Point2 p3 = p2 * 2.0; Point2 p4 = p3 + p2; Point2 p5 = p4 - p2 * 3; EXPECT(p5 == oo); } CASE("Point2 equality") { Point2 p1 = {1.0, 2.0}; Point2 p2 = {1.0, 2.0}; EXPECT(p1 == p2); } CASE("Point2 inequality") { Point2 p1 = {1.0, 3.0}; Point2 p2 = {1.0, 4.0}; EXPECT(p1 != p2); } CASE("Point2 comparison") { Point2 p1 = {2.0, 1.0}; Point2 p2 = {1.0, 2.0}; EXPECT(p2 < p1); } CASE("Point distance comparison") { Point2 p1 = {2.0, 1.0}; Point2 p2 = {1.0, 2.0}; EXPECT(types::is_approximately_equal(sqrt(2.0), p1.distance(p2))); Point2 p3 = {5.0, 5.0}; EXPECT(types::is_approximately_equal(p1.distance(p3), 5.0)); } CASE("Point distance2 comparison") { Point2 p1 = {2.0, 1.0}; Point2 p2 = {1.0, 2.0}; EXPECT(types::is_approximately_equal(p1.distance2(p2), 2.0)); Point2 p3 = {5.0, 5.0}; EXPECT(types::is_approximately_equal(p1.distance2(p3), 25.0)); } CASE("Point3 cross") { Point3 p1 = {1.0, 0.0, 0.0}; Point3 p2 = {0.0, 1.0, 0.0}; Point3 p3 = {0.0, 0.0, 1.0}; EXPECT(p3 == Point3::cross(p1, p2)); EXPECT(p1 == Point3::cross(p2, p3)); Point3 p4 = {1.0, 2.0, 3.0}; Point3 p5 = {-1.0, -2.0, 4.0}; Point3 p6 = {2.0, -1.0, 0.0}; Point3 p7 = Point3::normalize(Point3::cross(p4, p5)); Point3 p8 = Point3::normalize(p6); EXPECT(types::is_approximately_equal(p7[XX], p8[XX])); EXPECT(types::is_approximately_equal(p7[YY], p8[YY])); EXPECT(types::is_approximately_equal(p7[ZZ], p8[ZZ])); } } // namespace eckit::test //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/geometry/test_polygon.cc0000664000175000017500000003432015161702250021036 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geometry/Point2.h" #include "eckit/geometry/polygon/LonLatPolygon.h" #include "eckit/geometry/polygon/Polygon.h" #include "eckit/testing/Test.h" namespace eckit::test { CASE("Polygon") { using geometry::polygon::Polygon; SECTION("empty polygon") { Polygon poly1; Polygon poly2; EXPECT(poly1.sameAs(poly2)); } SECTION("equality") { Polygon poly1; Polygon poly2; EXPECT(poly1.num_vertices() == 0); Polygon::value_type p1 = {1.0, 2.0}; poly1.push_front(p1); EXPECT(poly1.num_vertices() == 1); EXPECT(poly1.vertex(0) == p1); EXPECT(!poly1.sameAs(poly2)); poly2.push_back(p1); EXPECT(poly1.sameAs(poly2)); Polygon::value_type p2 = {2.0, 1.0}; poly1.push_front(p2); EXPECT(!poly1.sameAs(poly2)); poly2.push_back(p2); EXPECT(!poly1.sameAs(poly2)); } SECTION("congruence") { Polygon poly1; Polygon poly2; EXPECT(poly1.congruent(poly2)); Polygon::value_type p1 = {1.0, 2.0}; poly1.push_front(p1); EXPECT(!poly1.congruent(poly2)); poly2.push_back(p1); EXPECT(poly1.congruent(poly2)); Polygon::value_type p2 = {2.0, 1.0}; poly1.push_front(p2); EXPECT(!poly1.congruent(poly2)); poly2.push_back(p2); EXPECT(poly1.congruent(poly2)); Polygon::value_type p3 = {3.0, 4.0}; poly2.push_back(p3); Polygon poly3 = {p2, p3, p1}; EXPECT(!poly2.sameAs(poly3)); EXPECT(poly2.congruent(poly3)); EXPECT(poly2.num_vertices() == 3); EXPECT(poly2.vertex(2) == poly3.vertex(1)); Polygon poly4 = {p3, p1, p2}; EXPECT(!poly2.sameAs(poly4)); EXPECT(poly2.congruent(poly4)); } } CASE("LonLatPolygon") { using Polygon = geometry::polygon::LonLatPolygon; SECTION("Construction") { const std::vector points1{{0, 0}, {1, 1}, {2, 2}, {0, 0}}; const std::vector points2{{0, 0}, {1, 0}, {2, 0}, {2, 1}, {2, 2}, {1, 2}, {0, 2}, {0, 1}, {0, 0}}; EXPECT_EQUAL(Polygon(points1).size(), 2); EXPECT_EQUAL(Polygon(points1.begin(), points1.end()).size(), 2); EXPECT_EQUAL(Polygon(points2).size(), 5); EXPECT_EQUAL(Polygon(points2.begin(), points2.end()).size(), 5); } SECTION("Contains North pole") { const std::vector points{{0, 90}, {0, 0}, {1, 0}, {1, 90}, {0, 90}}; Polygon poly1(points); EXPECT(poly1.contains({0, 90})); EXPECT(poly1.contains({10, 90})); EXPECT_NOT(poly1.contains({0, -90})); EXPECT_NOT(poly1.contains({10, -90})); Polygon poly2(points, false); EXPECT(poly2.contains({0, 90})); EXPECT_NOT(poly2.contains({10, 90})); EXPECT_NOT(poly2.contains({0, -90})); EXPECT_NOT(poly2.contains({10, -90})); } SECTION("Contains South pole") { const std::vector points{{0, -90}, {0, 0}, {1, 0}, {1, -90}, {0, -90}}; Polygon poly1(points); EXPECT_NOT(poly1.contains({0, 90})); EXPECT_NOT(poly1.contains({10, 90})); EXPECT(poly1.contains({0, -90})); EXPECT(poly1.contains({10, -90})); Polygon poly2(points, false); EXPECT_NOT(poly2.contains({0, 90})); EXPECT_NOT(poly2.contains({10, 90})); EXPECT(poly2.contains({0, -90})); EXPECT_NOT(poly2.contains({10, -90})); } SECTION("Contains South and North poles") { Polygon poly({{0, -90}, {0, 90}, {1, 90}, {1, -90}, {0, -90}}); EXPECT(poly.contains({0, 90})); EXPECT(poly.contains({10, 90})); EXPECT(poly.contains({0, 0})); EXPECT_NOT(poly.contains({10, 0})); EXPECT(poly.contains({0, -90})); EXPECT(poly.contains({10, -90})); } SECTION("MIR-566: wide polygon") { Polygon poly1({{0, 0}, {361, 0}, {361, 2}, {0, 2}, {0, 0}}); EXPECT(poly1.contains({0, 1})); EXPECT(poly1.contains({2, 1})); EXPECT(poly1.contains({362, 1})); EXPECT(poly1.contains({722, 1})); Polygon poly2({{0, 0}, {11, 0}, {11, 2}, {0, 2}, {0, 0}}); EXPECT(poly2.contains({0, 1})); EXPECT(poly2.contains({2, 1})); EXPECT(poly2.contains({362, 1})); EXPECT(poly2.contains({722, 1})); Polygon poly3({{0, 0}, {360, 0}, {360, 2}, {0, 2}, {0, 0}}); EXPECT(poly3.contains({0, 1})); EXPECT(poly3.contains({2 - 360, 1})); EXPECT(poly3.contains({2, 1})); EXPECT(poly3.contains({2 + 360, 1})); Polygon poly4({{-100, 18}, {21, 30}, {150, 50}, {260, 18}, {-100, 18}}); EXPECT(poly4.contains({-10 - 360, 18})); EXPECT(poly4.contains({-10, 18})); EXPECT(poly4.contains({-10 + 360, 18})); Polygon poly5({{-44.2299698513, 44.8732496764}, {-12.2849279262, 75.2545011911}, {72.2148603917, 76.7993105902}, {196.903572422, 71.1350094603}, {304.194105814, 52.8269579527}, {266.886210026, -17.7495991714}, {108.327652927, 34.8499103834}, {-96.2694736324, -17.4340627522}, {-99.8761719143, 7.28288763265}, {-44.2299698513, 44.8732496764}}); for (double lon = -1, lat = 10; lat < 70; lat += 1) { EXPECT(poly5.contains({lon - 360, lat})); EXPECT(poly5.contains({lon, lat})); EXPECT(poly5.contains({lon + 360, lat})); } constexpr double eps = 0.001; constexpr double globe = 360; Polygon poly6({{0 * globe, 4 + eps}, {1 * globe, 2 + eps}, {2 * globe, 0 + eps}, {3 * globe, -2 + eps}, {4 * globe, -4 + eps}, {4 * globe, -4 - eps}, {3 * globe, -2 - eps}, {2 * globe, 0 - eps}, {1 * globe, 2 - eps}, {0 * globe, 4 - eps}, {0 * globe, 4 + eps}}); const std::vector list_lons{-2. * globe, -globe, 0., globe, 2. * globe}; const std::vector list_lats1{4., 2., 0., -2.}; const std::vector list_lats2{5., 3., 1., -1., -3., -5.}; for (double lon : list_lons) { for (double lat : list_lats1) { EXPECT(poly6.contains({lon + 180., lat - 1.})); EXPECT(poly6.contains({lon, lat})); } for (double lat : list_lats2) { EXPECT_NOT(poly6.contains({lon, lat})); EXPECT_NOT(poly6.contains({lon + 180., lat - 1.})); } } // HEALPix-like equator wedge in longitude Polygon poly( {{0, 1}, {0, 90}, {360, 90}, {360, 1}, {361, 0}, {360, -1}, {360, -90}, {0, -90}, {0, -1}, {1, 0}, {0, 1}}); EXPECT(poly.contains({0, 0})); EXPECT(poly.contains({1, 0})); EXPECT(poly.contains({360, 0})); EXPECT(poly.contains({361, 0})); EXPECT(poly.contains({720, 0})); EXPECT(poly.contains({721, 0})); } SECTION("MIR-566: winding number strict check of edges") { Polygon poly({{110, -34}, {90, -62}, {100, -59}, {110, -50}, {132, -40}, {110, -34}}); EXPECT_NOT(poly.contains({90, -40})); EXPECT_NOT(poly.contains({90, -34})); } SECTION("Simple rectangular polygon") { double lonmin = 0; double lonmax = 360; double lonmid = 0.5 * (lonmin + lonmax); double latmax = 80; double latmin = 0; double latmid = 0.5 * (latmin + latmax); Polygon poly({{lonmin, latmax}, {lonmax, latmax}, {lonmax, latmin}, {lonmin, latmin}, {lonmin, latmax}}); // SUBSECTION: Contains edges { EXPECT(poly.contains({lonmin, latmax})); EXPECT(poly.contains({lonmid, latmax})); EXPECT(poly.contains({lonmax, latmax})); EXPECT(poly.contains({lonmax, latmid})); EXPECT(poly.contains({lonmax, latmin})); EXPECT(poly.contains({lonmid, latmin})); EXPECT(poly.contains({lonmin, latmin})); EXPECT(poly.contains({lonmin, latmid})); } // SUBSECTION: "Contains in/outward of edges" { constexpr auto eps = 0.001; for (size_t i = 0; i <= 100; ++i) { const auto lon = lonmin + static_cast(i) * (lonmax - lonmin) / 100.; EXPECT(poly.contains({lon, latmin + eps})); EXPECT(poly.contains({lon, latmax - eps})); EXPECT_NOT(poly.contains({lon, latmin - eps})); EXPECT_NOT(poly.contains({lon, latmax + eps})); const auto lat = latmin + static_cast(i) * (latmax - latmin) / 100.; EXPECT(poly.contains({lonmin + eps, lat})); EXPECT(poly.contains({lonmax - eps, lat})); EXPECT(poly.contains({lonmin - eps, lat})); // periodic EXPECT(poly.contains({lonmax + eps, lat})); // ... } } // Test points at non-canonical coordinates // Default behavior throws EXPECT_THROWS_AS(poly.contains({lonmid, 180. - latmid}), eckit::BadValue); EXPECT(poly.contains({lonmid + 360., latmid}, true)); EXPECT(poly.contains({lonmid, 180. - latmid}, true)); } SECTION("Parallelogram") { const std::vector points{{0, 0}, {1, 1}, {2, 1}, {1, 0}, {0, 0}}; Polygon poly(points); for (const auto& p : points) { EXPECT(poly.contains(p)); } EXPECT_NOT(poly.contains({0, 1})); EXPECT_NOT(poly.contains({2, 0})); } SECTION("Degenerate polygon") { const std::vector points{{0, 0}, {2, 0}, {2, 0} /*duplicate*/, {0, 2}, {0, 0}}; Polygon poly(points); for (const auto& p : points) { EXPECT(poly.contains(p)); } for (const auto& p : std::vector{{2, 2}}) { EXPECT_NOT(poly.contains(p)); } } SECTION("Self-intersecting polygon") { Polygon poly1({{-1, -1}, {1, 1}, {1, -1}, {-1, 1}, {-1, -1}}); EXPECT(poly1.contains({0, 0})); EXPECT(poly1.contains({-1, 0})); EXPECT(poly1.contains({1, 0})); EXPECT_NOT(poly1.contains({0, 1})); EXPECT_NOT(poly1.contains({0, -1})); Polygon poly2({{-1, -1}, {1, -1}, {-1, 1}, {1, 1}, {-1, -1}}); EXPECT(poly2.contains({0, 0})); EXPECT_NOT(poly2.contains({-1, 0})); EXPECT_NOT(poly2.contains({1, 0})); EXPECT(poly2.contains({0, 1})); EXPECT(poly2.contains({0, -1})); Polygon poly3({{-1, 89}, {1, 89}, {0, 90}, {181, 89}, {179, 89}, {0, 90}, {-1, 89}}); EXPECT(poly3.size() == 7); const std::vector list_lons{-720., -360., 0., 360., 720.}; for (const auto& lon : list_lons) { EXPECT(poly3.contains({lon, 89.})); EXPECT(poly3.contains({lon + 180, 89.})); EXPECT_NOT(poly3.contains({lon + 90, 89.})); EXPECT_NOT(poly3.contains({lon + 270, 89.})); } } SECTION("Partitioning (includePoles=false)") { auto mid = [](double a, double b) { return (a + b) / 2.; }; constexpr double lon[] = {0, 90, 180, 270, 360}; constexpr double lat[] = {90, 0, -90}; Polygon polys[] = { Polygon({{lon[0], lat[1]}, {lon[1], lat[1]}, {lon[1], lat[0]}, {lon[0], lat[0]}, {lon[0], lat[1]}}, false), Polygon({{lon[1], lat[1]}, {lon[2], lat[1]}, {lon[2], lat[0]}, {lon[1], lat[0]}, {lon[1], lat[1]}}, false), Polygon({{lon[2], lat[1]}, {lon[3], lat[1]}, {lon[3], lat[0]}, {lon[2], lat[0]}, {lon[2], lat[1]}}, false), Polygon({{lon[3], lat[1]}, {lon[4], lat[1]}, {lon[4], lat[0]}, {lon[3], lat[0]}, {lon[3], lat[1]}}, false), Polygon({{lon[0], lat[1]}, {lon[1], lat[1]}, {lon[1], lat[2]}, {lon[0], lat[2]}, {lon[0], lat[1]}}, false), Polygon({{lon[1], lat[1]}, {lon[2], lat[1]}, {lon[2], lat[2]}, {lon[1], lat[2]}, {lon[1], lat[1]}}, false), Polygon({{lon[2], lat[1]}, {lon[3], lat[1]}, {lon[3], lat[2]}, {lon[2], lat[2]}, {lon[2], lat[1]}}, false), Polygon({{lon[3], lat[1]}, {lon[4], lat[1]}, {lon[4], lat[2]}, {lon[3], lat[2]}, {lon[3], lat[1]}}, false)}; std::vector points; const std::vector list_lons{lon[0], mid(lon[0], lon[1]), lon[1], mid(lon[1], lon[2]), lon[2], mid(lon[2], lon[3]), lon[3], mid(lon[3], lon[4])}; const std::vector list_lats{lat[0], mid(lat[0], lat[1]), lat[1], mid(lat[1], lat[2]), lat[2]}; for (double lon : list_lons) { for (double lat : list_lats) { points.emplace_back(lon, lat); } } std::vector counts(points.size(), 0); for (size_t i = 0; i < points.size(); ++i) { for (const auto& poly : polys) { if (poly.contains(points[i])) { ++counts[i]; } } } for (size_t i = 0; i < counts.size(); i += list_lats.size() * 2) { EXPECT(counts[i + 0] == 2); EXPECT(counts[i + 1] == 2); EXPECT(counts[i + 2] == 4); EXPECT(counts[i + 3] == 2); EXPECT(counts[i + 4] == 2); EXPECT(counts[i + 5] == 1); EXPECT(counts[i + 6] == 1); EXPECT(counts[i + 7] == 2); EXPECT(counts[i + 8] == 1); EXPECT(counts[i + 9] == 1); } } } } // namespace eckit::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geometry/test_kpoint.cc0000664000175000017500000000452615161702250020660 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geometry/Point3.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; using namespace eckit::geometry; namespace eckit::test { CASE("KPoint Inits to Zero") { Point3 p; EXPECT(p[XX] == 0.); EXPECT(p[YY] == 0.); EXPECT(p[ZZ] == 0.); } CASE("KPoint Inits to Array") { Point3 p = {1.0, 2.0, 3.0}; EXPECT(p[XX] == 1.0); EXPECT(p[YY] == 2.0); EXPECT(p[ZZ] == 3.0); } CASE("KPoint addition") { Point3 p1 = {1.0, 2.0, 3.0}; Point3 p2 = {1.0, 2.0, 3.0}; Point3 r = p1 + p2; EXPECT(r[XX] == 2.0); EXPECT(r[YY] == 4.0); EXPECT(r[ZZ] == 6.0); } CASE("KPoint subtraction") { Point3 p1 = {2.0, 5.0, 7.0}; Point3 p2 = {1.0, 2.0, 3.0}; Point3 r = p1 - p2; EXPECT(r[XX] == 1.0); EXPECT(r[YY] == 3.0); EXPECT(r[ZZ] == 4.0); } CASE("KPoint subtraction") { Point3 p1 = {2.0, 5.0, 7.0}; Point3 p2 = {1.0, 2.0, 3.0}; Point3 r = p1 - p2; EXPECT(r[XX] == 1.0); EXPECT(r[YY] == 3.0); EXPECT(r[ZZ] == 4.0); } CASE("KPoint scaling") { Point3 p1 = {1.0, 2.0, 3.0}; Point3 r = p1 * 42.0; EXPECT(r[XX] == 42.0); EXPECT(r[YY] == 84.0); EXPECT(r[ZZ] == 126.0); } CASE("KPoint equality") { Point3 p1 = {1.0, 2.0, 3.0}; Point3 p2 = {1.0, 2.0, 3.0}; EXPECT(p1 == p2); } CASE("KPoint inequality") { Point3 p1 = {1.0, 2.0, 3.0}; Point3 p2 = {1.0, 2.0, 4.0}; EXPECT(p1 != p2); } CASE("KPoint comparison") { Point3 p1 = {2.0, 1.0, 0.0}; Point3 p2 = {1.0, 2.0, 4.0}; EXPECT(p2 < p1); } CASE("KPoint distance2 comparison") { Point3 zz; Point3 p1 = {2.0, 1.0, 0.0}; Point3 p2 = {1.0, 2.0, 4.0}; EXPECT(p1.distance2(zz) < p2.distance2(zz)); } } // namespace eckit::test //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/geometry/CMakeLists.txt0000664000175000017500000000037315161702250020542 0ustar alastairalastairforeach( _test coordinate_helpers great_circle kdtree sphere kpoint points polygon ) ecbuild_add_test( TARGET eckit_test_geometry_${_test} SOURCES test_${_test}.cc LIBS eckit_geometry ) endforeach() eckit-2.0.7/tests/geo/0000775000175000017500000000000015161702250014716 5ustar alastairalastaireckit-2.0.7/tests/geo/grid_regular_gg.cc0000664000175000017500000001032115161702250020345 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/Grid.h" #include "eckit/geo/grid/regular/RegularGaussian.h" #include "eckit/geo/util.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { using grid::regular::RegularGaussian; CASE("sizes") { struct test_t { explicit test_t(size_t N) : N(N), size(4 * N * 2 * N) {} size_t N; size_t size; } tests[]{test_t{2}, test_t{3}, test_t{64}}; for (const auto& test : tests) { std::unique_ptr grid1(GridFactory::build(spec::Custom({{"grid", "f" + std::to_string(test.N)}}))); std::unique_ptr grid2(GridFactory::build(spec::Custom({{"type", "regular_gg"}, {"N", test.N}}))); RegularGaussian grid3(test.N); EXPECT(grid1->size() == test.size); EXPECT(grid2->size() == test.size); EXPECT(grid3.size() == test.size); } } CASE("points") { RegularGaussian grid(1); const std::vector ref{ {0., 35.264389683}, {90., 35.264389683}, {180., 35.264389683}, {270., 35.264389683}, // {0., -35.264389683}, {90., -35.264389683}, {180., -35.264389683}, {270., -35.264389683}, }; auto points = grid.to_points(); EXPECT(points.size() == grid.size()); ASSERT(points.size() == ref.size()); auto it = grid.begin(); for (size_t i = 0; i < points.size(); ++i) { EXPECT(points_equal(ref[i], points[i])); EXPECT(points_equal(ref[i], *it)); ++it; } EXPECT(it == grid.end()); size_t i = 0; for (const auto& it : grid) { EXPECT(points_equal(ref[i++], it)); } EXPECT(i == grid.size()); } CASE("crop") { spec::Custom a({{"grid", "f2"}}); std::unique_ptr grid1(GridFactory::build(a)); auto n1 = grid1->size(); EXPECT_EQUAL(n1, 32); a.set("south", 0.); std::unique_ptr grid2(GridFactory::build(a)); auto n2 = grid2->size(); EXPECT_EQUAL(n2, n1 / 2); spec::Custom b{{{"grid", "f2"}, {"west", -180}}}; std::unique_ptr grid3(GridFactory::build(b)); auto n3 = grid3->size(); EXPECT_EQUAL(n3, n1); auto bbox3 = grid3->boundingBox(); EXPECT(bbox3.periodic()); bbox3 = {bbox3.north(), bbox3.west(), bbox3.south(), 0.}; EXPECT_NOT(bbox3.periodic()); std::unique_ptr grid4(grid3->make_grid_cropped(bbox3)); auto n4 = grid4->size(); EXPECT_EQUAL(n4, 5 * 4); // Ni * Nj b.set("east", -1.); std::unique_ptr grid5(GridFactory::build(b)); auto n5 = grid5->size(); EXPECT_EQUAL(n5, 4 * 4); // Ni * Nj const std::vector ref{ {-180., 59.444408289}, {-135., 59.444408289}, {-90., 59.444408289}, {-45., 59.444408289}, {-180., 19.875719147}, {-135., 19.875719147}, {-90., 19.875719147}, {-45., 19.875719147}, {-180., -19.875719147}, {-135., -19.875719147}, {-90., -19.875719147}, {-45., -19.875719147}, {-180., -59.444408289}, {-135., -59.444408289}, {-90., -59.444408289}, {-45., -59.444408289}, }; auto points5 = grid5->to_points(); EXPECT(points5.size() == grid5->size()); ASSERT(points5.size() == ref.size()); auto it = grid5->begin(); for (size_t i = 0; i < points5.size(); ++i) { EXPECT(points_equal(ref[i], points5[i])); EXPECT(points_equal(ref[i], *it)); ++it; } EXPECT(it == grid5->end()); size_t i = 0; for (const auto& it : *grid5) { EXPECT(points_equal(ref[i++], it)); } EXPECT_EQUAL(i, n5); } CASE("equals") { std::unique_ptr grid1(GridFactory::build(spec::Custom({{"grid", "f3"}}))); std::unique_ptr grid2(new RegularGaussian(3)); EXPECT(*grid1 == *grid2); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/point2.cc0000664000175000017500000000433015161702250016440 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2. * which can be obtained at http://www.apache.org/licenses/LICENSE-2.. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/PointXY.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::test { CASE("PointXY initialisation") { PointXY z; EXPECT(z.X() == 0.); EXPECT(z.Y() == 0.); PointXY q{4., 5.}; EXPECT(q.X() == 4.); EXPECT(q.Y() == 5.); PointXY r(q); EXPECT(r.X() == 4.); EXPECT(r.Y() == 5.); } CASE("PointXY addition") { PointXY p1{1., 2.}; PointXY p2{2., 4.}; PointXY r = p1 + p2; EXPECT(r.X() == 3.); EXPECT(r.Y() == 6.); } CASE("PointXY subtraction") { PointXY p1{2., 5.}; PointXY p2{1., 2.}; PointXY r = p1 - p2; EXPECT(r.X() == 1.); EXPECT(r.Y() == 3.); } CASE("PointXY scaling") { PointXY p1{1., 2.}; PointXY p2(p1); PointXY r = p1 * 42.; EXPECT(r.X() == 42.); EXPECT(r.Y() == 84.); PointXY oo; PointXY p3 = p2 * 2.; PointXY p4 = p3 + p2; PointXY p5 = p4 - p2 * 3; EXPECT(p5 == oo); } CASE("PointXY equality") { PointXY p1{1., 2.}; PointXY p2{1., 2.}; EXPECT(p1 == p2); } CASE("PointXY inequality") { PointXY p1{1., 3.}; PointXY p2{1., 4.}; EXPECT(p1 != p2); } CASE("PointXY distance comparison") { PointXY p1{2., 1.}; PointXY p2{1., 2.}; PointXY p3{5., 5.}; EXPECT(types::is_approximately_equal(std::sqrt(2.), p1.distance(p2))); EXPECT(types::is_approximately_equal(5., p1.distance(p3))); } CASE("PointXY distance2 comparison") { PointXY p1{2., 1.}; PointXY p2{1., 2.}; PointXY p3{5., 5.}; EXPECT(types::is_approximately_equal(p1.distance2(p2), 2.)); EXPECT(types::is_approximately_equal(p1.distance2(p3), 25.)); EXPECT(p2.distance2(p1) < p3.distance2(p1)); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/great_circle.cc0000664000175000017500000002061015161702250017647 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * (C) Crown Copyright 2023 Met Office. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/geo/GreatCircle.h" #include "eckit/geo/PointLonLat.h" #include "eckit/testing/Test.h" #define EXPECT_APPROX(a, b, eps) EXPECT(::eckit::types::is_approximately_equal((a), (b), (eps))) namespace eckit::geo::test { const PointLonLat VALPARAISO(-71.6, -33.); const PointLonLat SHANGHAI(121.8, 31.4); CASE("great circle intersections") { using types::is_approximately_equal; using types::is_approximately_greater_or_equal; auto is_approximately_equal_longitude = [](double lon1, double lon2, double epsilon = std::numeric_limits::epsilon()) -> bool { while (lon2 < lon1) { lon2 += 360; } while (lon1 >= lon1 + 360) { lon2 -= 360; } return is_approximately_equal(lon1, lon2, epsilon) || is_approximately_equal(lon1, lon2 - 360, epsilon); }; auto is_approximately_pole = [](double lat, double epsilon = std::numeric_limits::epsilon()) -> bool { return is_approximately_equal(std::abs(lat), 90., epsilon); }; auto is_approximately_equator = [](double lat, double epsilon = std::numeric_limits::epsilon()) -> bool { return is_approximately_equal(lat, 0., epsilon); }; const std::vector latitudes{ 90, 60, 45, 30, 0, -30, -45, -60, -90, }; const std::vector longitudes{ -181, -180, -135, -90, -45, 0, 45, 90, 135, 180, 225, 270, 315, 360, 361, }; const std::vector antipodes{ {0, 0}, {180, 0}, {-180, 0}, {0, 0}, {-90, 0}, {90, 0}, {90, 0}, {-90, 0}, {0, 90}, {0, -90}, {0, -90}, {0, 90}, {45, 45}, {225, -45}, }; SECTION("example intersection with meridian and parallel") { // latitude at Valparaíso-Shanghai mid-point GreatCircle gc(VALPARAISO, SHANGHAI); const PointLonLat mid(-159.18, -6.81); auto lats = gc.latitude(mid.lon()); EXPECT(lats.size() == 1 && is_approximately_equal(lats[0], mid.lat(), 0.01)); auto lons = gc.longitude(mid.lat()); EXPECT(lons.size() == 2); EXPECT(is_approximately_equal_longitude(lons[0], mid.lon(), 0.01) || is_approximately_equal_longitude(lons[1], mid.lon(), 0.01)); } SECTION("mal-formed great circle") { for (size_t i = 0; i < antipodes.size(); i += 2) { const PointLonLat& A(antipodes[i]); const PointLonLat& B(antipodes[i + 1]); EXPECT_THROWS_AS(GreatCircle(A, A), BadValue); EXPECT_THROWS_AS(GreatCircle(B, B), BadValue); EXPECT_THROWS_AS(GreatCircle(A, B), BadValue); if (is_approximately_pole(A.lat())) { for (double lon1_gc : longitudes) { for (double lon2_gc : longitudes) { EXPECT_THROWS_AS(GreatCircle({lon1_gc, A.lat()}, {lon2_gc, A.lat()}), BadValue); EXPECT_THROWS_AS(GreatCircle({lon1_gc, B.lat()}, {lon2_gc, B.lat()}), BadValue); } } } } } SECTION("intersection at quadrants") { for (double lat_gc : latitudes) { if (!is_approximately_pole(lat_gc) && !is_approximately_equator(lat_gc)) { for (double lon_gc : longitudes) { GreatCircle gc({lon_gc, lat_gc}, {lon_gc + 90, 0}); EXPECT(!gc.crossesPoles()); auto lon_at_equator = gc.longitude(0); EXPECT(lon_at_equator.size() == 2); EXPECT((is_approximately_equal_longitude(lon_gc + 90, lon_at_equator[0]) && is_approximately_equal_longitude(lon_gc - 90, lon_at_equator[1])) || (is_approximately_equal_longitude(lon_gc - 90, lon_at_equator[0]) && is_approximately_equal_longitude(lon_gc + 90, lon_at_equator[1]))); auto lon_extrema1 = gc.longitude(lat_gc); EXPECT(lon_extrema1.size() == 1 && is_approximately_equal_longitude(lon_extrema1[0], lon_gc, 0.01)); auto lon_extrema2 = gc.longitude(-lat_gc); EXPECT(lon_extrema2.size() == 1 && is_approximately_equal_longitude(lon_extrema2[0], lon_gc + 180, 0.01)); } } } } SECTION("intersection with parallels when crossing the poles") { for (double lon : longitudes) { for (double lat : latitudes) { { GreatCircle gc({lon, -10}, {lon, 10}); EXPECT(gc.crossesPoles()); auto lons = gc.longitude(lat); size_t N = is_approximately_pole(lat) ? 1 : 2; EXPECT(lons.size() == N); if (N == 1) { EXPECT(is_approximately_equal_longitude(lons[0], lon)); } else { EXPECT(is_approximately_equal_longitude(lons[0] + 180, lons[1])); EXPECT(is_approximately_equal_longitude(lons[0], lon) || is_approximately_equal_longitude(lons[1], lon)); } } if (!is_approximately_pole(lat) && !is_approximately_equator(lat)) { GreatCircle gc({lon, lat}, {lon + 180, lat}); EXPECT(gc.crossesPoles()); auto lons = gc.longitude(lat); EXPECT(lons.size() == 2); EXPECT(is_approximately_equal_longitude(lons[0] + 180, lons[1])); EXPECT(is_approximately_equal_longitude(lons[0], lon) || is_approximately_equal_longitude(lons[1], lon)); } } } } SECTION("intersection with parallels") { for (double lat_gc : latitudes) { if (/* avoid mal-forming */ !is_approximately_pole(lat_gc)) { for (double lat : latitudes) { GreatCircle gc({-1, lat_gc}, {1, lat_gc}); EXPECT(!gc.crossesPoles()); auto lons = gc.longitude(lat); size_t N = is_approximately_equator(lat_gc) ? 0 : is_approximately_greater_or_equal(std::abs(lat_gc), std::abs(lat)) ? 2 : 0; EXPECT(lons.size() == N); for (auto lon : lons) { auto lats = gc.latitude(lon); EXPECT(lats.size() == 1 && is_approximately_equal(lats[0], lat, 0.01)); } } } } } SECTION("equator great circle intersection with meridian and parallel") { for (double lon : longitudes) { GreatCircle eq({lon - 1, 0}, {lon + 1, 0}); EXPECT(!eq.crossesPoles()); // non-intersection with parallels for (double lat : latitudes) { EXPECT(eq.longitude(lat).empty()); } // intersect one latitude only, for specific longitudes auto lats = eq.latitude(lon); EXPECT(lats.size() == 1 && is_approximately_equator(lats[0])); } } } CASE("great circle course") { SECTION("Valparaíso-Shanghai") { GreatCircle gc(VALPARAISO, SHANGHAI); const auto [course1, course2] = gc.course(); EXPECT_APPROX(-94.41, course1, 0.01); EXPECT_APPROX(-78.42, course2, 0.01); } SECTION("polar") { GreatCircle gc({0., 89.}, {180., 89.}); const auto [course3, course4] = gc.course(); EXPECT_APPROX(0., course3, 1.e-14); EXPECT_APPROX(180., std::abs(course4), 1.e-14); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/projection_proj.cc0000664000175000017500000001117715161702250020442 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/Projection.h" #include "eckit/geo/area/BoundingBox.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { using P = std::unique_ptr; CASE("projection: proj") { constexpr double eps = 1e-6; if (ProjectionFactoryType::instance().exists("proj")) { PointLonLat a{12., 55.}; struct { const Point b; const std::string target; } tests_proj[] = { {PointXY{691875.632137542, 6098907.825129169}, "+proj=utm +zone=32 +datum=WGS84"}, {PointXY{691875.632137542, 6098907.825129169}, "EPSG:32632"}, {a, "EPSG:4326"}, {a, "EPSG:4979"}, {PointXYZ{3586469.6567764, 762327.65877826, 5201383.5232023}, "EPSG:4978"}, {PointXYZ{3574529.7050235, 759789.74368715, 5219005.2599833}, "+proj=cart +R=6371229."}, {PointXYZ{3574399.5431832, 759762.07693392, 5218815.216709}, "+proj=cart +ellps=sphere"}, {a, "+proj=latlon +ellps=sphere"}, }; for (const auto& test : tests_proj) { P projection(ProjectionFactoryType::instance().get("proj").create( spec::Custom{{{"source", "EPSG:4326"}, {"target", test.target}}})); #if 0 Log::info() << "ellipsoid: '" << PROJ::ellipsoid(projection.target()) << std::endl; #endif auto b = projection->fwd(a); auto c = projection->inv(b); EXPECT(points_equal(b, test.b, eps)); EXPECT(points_equal(c, a, eps)); P reverse(ProjectionFactoryType::instance().get("proj").create( spec::Custom({{"source", test.target}, {"target", "EPSG:4326"}}))); auto d = reverse->fwd(test.b); auto e = reverse->inv(d); EXPECT(points_equal(d, a, eps)); EXPECT(points_equal(e, test.b, eps)); } P polar_stereographic_north(ProjectionFactoryType::instance().get("proj").create( spec::Custom{{{"source", "EPSG:4326"}, {"target", "+proj=stere +lat_0=90. +lon_0=-30. +R=6371229."}}})); P polar_stereographic_south(ProjectionFactoryType::instance().get("proj").create( spec::Custom{{{"source", "EPSG:4326"}, {"target", "+proj=stere +lat_0=-90. +lon_0=-30. +R=6371229."}}})); struct { const P& projection; const PointXY min; const PointXY max; const bool periodic; const bool contains_north_pole; const bool contains_south_pole; } tests_bbox[] = { {polar_stereographic_north, {-2e6, -2e6}, {2e6, 2e6}, true, true, false}, {polar_stereographic_north, {-2e6, -2e6}, {1e6, 1e6}, true, true, false}, {polar_stereographic_north, {-2e6, -2e6}, {-1e6, -1e6}, false, false, false}, {polar_stereographic_north, {-1e6, -1e6}, {2e6, 2e6}, true, true, false}, {polar_stereographic_north, {-1e6, -1e6}, {1e6, 1e6}, true, true, false}, {polar_stereographic_north, {1e6, 1e6}, {2e6, 2e6}, false, false, false}, {polar_stereographic_south, {-2e6, -2e6}, {2e6, 2e6}, true, false, true}, {polar_stereographic_south, {-2e6, -2e6}, {1e6, 1e6}, true, false, true}, {polar_stereographic_south, {-2e6, -2e6}, {-1e6, -1e6}, false, false, false}, {polar_stereographic_south, {-1e6, -1e6}, {2e6, 2e6}, true, false, true}, {polar_stereographic_south, {-1e6, -1e6}, {1e6, 1e6}, true, false, true}, {polar_stereographic_south, {1e6, 1e6}, {2e6, 2e6}, false, false, false}, }; for (const auto& test : tests_bbox) { auto bbox = area::BoundingBox::make_from_projection(test.min, test.max, *test.projection); ASSERT(bbox); EXPECT_EQUAL(test.periodic, bbox->periodic()); EXPECT_EQUAL(test.contains_north_pole, bbox->contains(NORTH_POLE)); EXPECT_EQUAL(test.contains_south_pole, bbox->contains(SOUTH_POLE)); auto global = test.periodic && test.contains_north_pole && test.contains_south_pole; EXPECT(global == bbox->global()); } } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/gaussian.cc0000664000175000017500000000377115161702250017047 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/util.h" #include "eckit/maths/FloatingPointExceptions.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" #define EXPECT_APPROX(a, b) EXPECT(::eckit::types::is_approximately_equal((a), (b), 1e-6)) namespace eckit::geo::test { /** * This case verifies that Floating Point Exceptions (FPEs) don't trigger for the functions: * - eckit::geo::util::gaussian_latitudes * - eckit::geo::util::gaussian_quadrature_weights * * FPE trapping is enabled to ensure any FPE-triggering operations are caught. * @see ECKIT-674 */ CASE("gaussian_latitudes") { const auto& lats_1 = geo::util::gaussian_latitudes(1, true); EXPECT(lats_1.size() == 2); EXPECT_APPROX(lats_1.front(), -35.264390); EXPECT_APPROX(lats_1.front(), -lats_1.back()); const auto& lats_2 = geo::util::gaussian_latitudes(80, false); EXPECT(lats_2.size() == 2 * 80); EXPECT_APPROX(lats_2.front(), 89.141519); EXPECT_APPROX(lats_2.front(), -lats_2.back()); } CASE("gaussian_quadrature_weights") { const auto& quad_1 = geo::util::gaussian_quadrature_weights(1); EXPECT(quad_1.size() == 2); EXPECT_APPROX(quad_1.front(), 0.5); EXPECT_APPROX(quad_1.front(), quad_1.back()); const auto& quad_2 = geo::util::gaussian_quadrature_weights(80); EXPECT(quad_2.size() == 2 * 80); EXPECT_APPROX(quad_2.front(), 0.000144); EXPECT_APPROX(quad_2.front(), quad_2.back()); } } // namespace eckit::geo::test int main(int argc, char** argv) { eckit::maths::FloatingPointExceptions::enable_floating_point_exceptions(); return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/integration.cc0000664000175000017500000001560215161702250017554 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to itr by virtue of its status as an intergovernmental organisation nor * does itr submit to any jurisdiction. */ #include #include #include #include "eckit/geo/eckit_geo_config.h" #include "eckit/testing/Test.h" #include "eckit/geo/Grid.h" namespace eckit::geo::test { using eckit::geo::Grid; CASE("gridSpec") { struct test_type { const std::string spec; const std::vector shape; }; SECTION("regular lat/lon") { std::unique_ptr grid1(GridFactory::make_from_string("grid: 2/1.0")); EXPECT(grid1->spec_str() == R"({"grid":[2,1]})"); std::unique_ptr grid2(GridFactory::make_from_string("grid: .250/001e-1")); EXPECT(grid2->spec_str() == R"({"grid":[0.25,0.1]})"); std::vector tests{ {R"({"grid":[0.05,0.05]})", {3601, 7200}}, {R"({"grid":[0.1,0.1]})", {1801, 3600}}, {R"({"grid":[0.125,0.125]})", {1441, 2880}}, {R"({"grid":[0.15,0.15]})", {1201, 2400}}, {R"({"grid":[0.2,0.2]})", {901, 1800}}, {R"({"grid":[0.25,0.25]})", {721, 1440}}, {R"({"grid":[0.3,0.3]})", {601, 1200}}, {R"({"grid":[0.35,0.35]})", {515, 1029}}, {R"({"grid":[0.4,0.4]})", {451, 900}}, {R"({"grid":[0.5,0.5]})", {361, 720}}, {R"({"grid":[0.6,0.6]})", {301, 600}}, {R"({"grid":[0.7,0.7]})", {257, 515}}, {R"({"grid":[0.75,0.75]})", {241, 480}}, {R"({"grid":[0.8,0.8]})", {225, 450}}, {R"({"grid":[0.9,0.9]})", {201, 400}}, {R"({"grid":[1,1]})", {181, 360}}, {R"({"grid":[1.2,1.2]})", {151, 300}}, {R"({"grid":[1.25,1.25]})", {145, 288}}, {R"({"grid":[1.4,1.4]})", {129, 258}}, {R"({"grid":[1.5,1.5]})", {121, 240}}, {R"({"grid":[1.6,1.6]})", {113, 225}}, {R"({"grid":[1.8,1.8]})", {101, 200}}, {R"({"grid":[2,2]})", {91, 180}}, {R"({"grid":[2.5,2.5]})", {73, 144}}, {R"({"grid":[5,5]})", {37, 72}}, {R"({"grid":[10,10]})", {19, 36}}, {R"({"grid":[30,30]})", {7, 12}}, }; for (const auto& t : tests) { std::unique_ptr grid(GridFactory::make_from_string(t.spec)); EXPECT(grid->spec_str() == t.spec); EXPECT(grid->shape() == t.shape); } } SECTION("HEALPix") { std::vector tests{ {R"({"grid":"H2","order":"nested"})", {48}}, {R"({"grid":"H2"})", {48}}, {R"({"grid":"H4","order":"nested"})", {192}}, {R"({"grid":"H4"})", {192}}, {R"({"grid":"H8","order":"nested"})", {768}}, {R"({"grid":"H8"})", {768}}, {R"({"grid":"H16","order":"nested"})", {3072}}, {R"({"grid":"H16"})", {3072}}, {R"({"grid":"H32","order":"nested"})", {12288}}, {R"({"grid":"H32"})", {12288}}, {R"({"grid":"H64","order":"nested"})", {49152}}, {R"({"grid":"H64"})", {49152}}, {R"({"grid":"H128","order":"nested"})", {196608}}, {R"({"grid":"H128"})", {196608}}, {R"({"grid":"H256","order":"nested"})", {786432}}, {R"({"grid":"H256"})", {786432}}, {R"({"grid":"H512","order":"nested"})", {3145728}}, {R"({"grid":"H512"})", {3145728}}, {R"({"grid":"H1024","order":"nested"})", {12582912}}, {R"({"grid":"H1024"})", {12582912}}, }; for (const auto& t : tests) { std::unique_ptr grid(GridFactory::make_from_string(t.spec)); EXPECT(grid->spec_str() == t.spec); EXPECT(grid->shape() == t.shape); } } SECTION("Gaussian grids") { std::vector tests{ {R"({"grid":"N32"})", {6114}}, {R"({"grid":"N48"})", {13280}}, {R"({"grid":"N64"})", {23112}}, {R"({"grid":"N80"})", {35718}}, {R"({"grid":"N96"})", {50662}}, {R"({"grid":"N128"})", {88838}}, {R"({"grid":"N160"})", {138346}}, {R"({"grid":"N200"})", {213988}}, {R"({"grid":"N256"})", {348528}}, {R"({"grid":"N320"})", {542080}}, {R"({"grid":"N400"})", {843490}}, {R"({"grid":"N512"})", {1373624}}, {R"({"grid":"N640"})", {2140702}}, {R"({"grid":"N1024"})", {5447118}}, {R"({"grid":"N1280"})", {8505906}}, {R"({"grid":"O32"})", {5248}}, {R"({"grid":"O48"})", {10944}}, {R"({"grid":"O64"})", {18688}}, {R"({"grid":"O80"})", {28480}}, {R"({"grid":"O96"})", {40320}}, {R"({"grid":"O128"})", {70144}}, {R"({"grid":"O160"})", {108160}}, {R"({"grid":"O200"})", {167200}}, {R"({"grid":"O256"})", {271360}}, {R"({"grid":"O320"})", {421120}}, {R"({"grid":"O400"})", {654400}}, {R"({"grid":"O512"})", {1067008}}, {R"({"grid":"O640"})", {1661440}}, {R"({"grid":"O1024"})", {4231168}}, {R"({"grid":"O1280"})", {6599680}}, {R"({"grid":"O2560"})", {26306560}}, }; for (const auto& t : tests) { std::unique_ptr grid(GridFactory::make_from_string(t.spec)); EXPECT(grid->spec_str() == t.spec); EXPECT(grid->shape() == t.shape); } } if (eckit_HAVE_ECKIT_CODEC) { SECTION("gridSpec (FESOM)") { std::vector tests{ {R"({"grid":"pi_N"})", {3140}}, }; for (const auto& t : tests) { std::unique_ptr grid(GridFactory::make_from_string(t.spec)); EXPECT(grid->spec_str() == t.spec); EXPECT(grid->shape() == t.shape); } } SECTION("gridSpec (ICON)") { std::vector tests{ {R"({"grid":"icon-grid-0055-r02b05-n"})", {2656}}, }; for (const auto& t : tests) { std::unique_ptr grid(GridFactory::make_from_string(t.spec)); EXPECT(grid->spec_str() == t.spec); EXPECT(grid->shape() == t.shape); } } SECTION("gridSpec (ORCA)") { std::vector tests{ {R"({"grid":"ORCA2_T"})", {182, 149}}, // eORCA025_T [1740494] is too large as an integration test }; for (const auto& t : tests) { std::unique_ptr grid(GridFactory::make_from_string(t.spec)); EXPECT(grid->spec_str() == t.spec); EXPECT(grid->shape() == t.shape); } } } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/grid_fesom.ek0000777000175000017500000000000015161702250030731 2bdc49d97a27e389fb86decd08a185c2f-447c17c8b247e53953901bb3bbd5d182.ekustar alastairalastaireckit-2.0.7/tests/geo/grid_orca.ek0000777000175000017500000000000015161702250031033 2d5bde4f52ff3a9bea5629cd9ac514410-313218f18f94c235fd8ef2d6cc6ad6fe.ekustar alastairalastaireckit-2.0.7/tests/geo/area_boundingbox.cc0000664000175000017500000001071215161702250020534 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/geo/area/BoundingBox.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("AreaFactory::make_from_string") { const area::BoundingBox expected_area; const std::string expected_spec = expected_area.spec_str(); for (const auto& spec : std::vector{ "{}", "{north: 90, south: -90, west: 0, east: 360}", "{type: bounding_box}", "{north: 90}", }) { std::unique_ptr area(geo::AreaFactory::make_from_string(spec)); EXPECT(expected_spec == area->spec_str()); EXPECT(expected_area == *area); } std::unique_ptr area1( geo::AreaFactory::make_from_string(R"({north: 90, west: 0, south: -90, east: 360})")); EXPECT(area1->spec_str() == expected_spec); std::unique_ptr area2( geo::AreaFactory::make_from_string(R"({north: 10, west: 35641, south: 0, east: 15130})")); EXPECT(area2->spec_str() == R"({"area":[10,1,0,10]})"); } CASE("global") { area::BoundingBox a; area::BoundingBox b(90, 0, -90, 360); EXPECT(a == b); } CASE("latitude (checks)") { EXPECT_THROWS(area::BoundingBox(-90, 0, 90, 360)); // fails South<=North EXPECT_NO_THROW(area::BoundingBox(90, 0, 90, 360)); EXPECT_NO_THROW(area::BoundingBox(-90, 0, -90, 360)); } CASE("longitude (normalisation)") { for (double west : {-900, -720, -540, -360, -180, 0, 180, 360, 540, 720, 900}) { area::BoundingBox a(90, west, 90, west); EXPECT_EQUAL(a.west(), west); EXPECT(a.empty()); area::BoundingBox b{90, west, -90, west - 1}; auto c = area::BoundingBox::make_from_area(90, west + 42 * 360., -90, west - 42 * 360. - 1); ASSERT(c); EXPECT(c->east() == c->west() + 360 - 1); EXPECT(b == *c); } } CASE("assignment") { area::BoundingBox a(10, 1, -10, 100); area::BoundingBox b(20, 2, -20, 200); EXPECT_NOT_EQUAL(a.north(), b.north()); EXPECT(a != b); b = a; EXPECT_EQUAL(a.north(), b.north()); EXPECT(a == b); b = {30., b.west(), b.south(), b.east()}; EXPECT_EQUAL(b.north(), 30); EXPECT_EQUAL(a.north(), 10); area::BoundingBox c(a); EXPECT_EQUAL(a.north(), c.north()); EXPECT(a == c); c = {40., c.west(), c.south(), c.east()}; EXPECT_EQUAL(c.north(), 40); EXPECT_EQUAL(a.north(), 10); auto d(std::move(a)); EXPECT_EQUAL(d.north(), 10); d = {50., d.west(), d.south(), d.east()}; EXPECT_EQUAL(d.north(), 50); } CASE("comparison") { area::BoundingBox a(10, 1, -10, 100); area::BoundingBox b(20, 2, -20, 200); EXPECT(!area::bounding_box_equal(a, b)); for (const auto& c : {a, b}) { const area::BoundingBox d{c.north(), c.west() + 42 * PointLonLat::FULL_ANGLE, c.south(), c.east() + 41 * PointLonLat::FULL_ANGLE}; EXPECT(area::bounding_box_equal(c, d)); } } CASE("properties") { area::BoundingBox a{10, 1, -10, 100}; area::BoundingBox b{20, 2, -20, 200}; auto c = area::BoundingBox::make_global_prime(); ASSERT(c); auto d = area::BoundingBox::make_global_antiprime(); ASSERT(c); area::BoundingBox e; for (const auto& bb : {a, b, *c, *d, e}) { EXPECT(!bb.empty()); EXPECT(bb.contains(PointLonLat{10, 0})); EXPECT(bb.global() == bb.contains(PointLonLat{0, 0})); EXPECT(bb.global() == (bb.periodic() && bb.contains(NORTH_POLE) && bb.contains(SOUTH_POLE))); } } CASE("intersects") { area::BoundingBox a(10, 1, -10, 100); area::BoundingBox b(20, 2, -20, 200); EXPECT(!area::bounding_box_equal(a, b)); for (const auto& c : {a, b}) { const area::BoundingBox d{c.north(), c.west() + 42 * PointLonLat::FULL_ANGLE, c.south(), c.east() + 41 * PointLonLat::FULL_ANGLE}; EXPECT(area::bounding_box_equal(c, d)); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/grid_icon.ek0000777000175000017500000000000015161702250035662 2eckit_geo_cache/grid/icon/e234e01a8556e9a84bcb42361d2f24e0-7efedd50f7ea0fe6e382b220a3bc68dd.ekustar alastairalastaireckit-2.0.7/tests/geo/grid_reduced_gg.cc0000664000175000017500000001374415161702250020333 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/grid/reduced/ReducedGaussian.h" #include "eckit/geo/util.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { using grid::reduced::ReducedGaussian; CASE("gridspec") { // different ways to instantiate the same grid (O2) for (const auto& spec : { spec::Custom({{"grid", "o2"}}), spec::Custom({{"N", 2}}), spec::Custom({{"pl", pl_type{20, 24, 24, 20}}}), }) { std::unique_ptr grid1(GridFactory::build(spec)); auto n1 = grid1->size(); EXPECT_EQUAL(n1, 88); EXPECT_EQUAL(grid1->spec_str(), R"({"grid":"O2"})"); spec::Custom hemisphere(spec.container()); hemisphere.set("south", 0); std::unique_ptr grid2(GridFactory::build(hemisphere)); auto n2 = grid2->size(); EXPECT_EQUAL(n2, n1 / 2); EXPECT_EQUAL(grid2->spec_str(), R"({"area":[90,0,0,360],"grid":"O2"})"); } } CASE("sizes") { struct test_t { explicit test_t(size_t N) : N(N), size(4 * N * (N + 9)) {} size_t N; size_t size; } tests[]{test_t{2}, test_t{3}, test_t{64}}; for (const auto& test : tests) { std::unique_ptr grid1(GridFactory::build(spec::Custom({{"grid", "o" + std::to_string(test.N)}}))); std::unique_ptr grid2(GridFactory::build(spec::Custom({{"type", "reduced_gg"}, {"N", test.N}}))); ReducedGaussian grid3(test.N); EXPECT(grid1->size() == test.size); EXPECT(grid2->size() == test.size); EXPECT(grid3.size() == test.size); } } CASE("points") { ReducedGaussian grid(1); const std::vector ref{ {0., 35.264389683}, {18., 35.264389683}, {36., 35.264389683}, {54., 35.264389683}, {72., 35.264389683}, {90., 35.264389683}, {108., 35.264389683}, {126., 35.264389683}, {144., 35.264389683}, {162., 35.264389683}, {180., 35.264389683}, {198., 35.264389683}, {216., 35.264389683}, {234., 35.264389683}, {252., 35.264389683}, {270., 35.264389683}, {288., 35.264389683}, {306., 35.264389683}, {324., 35.264389683}, {342., 35.264389683}, {0., -35.264389683}, {18., -35.264389683}, {36., -35.264389683}, {54., -35.264389683}, {72., -35.264389683}, {90., -35.264389683}, {108., -35.264389683}, {126., -35.264389683}, {144., -35.264389683}, {162., -35.264389683}, {180., -35.264389683}, {198., -35.264389683}, {216., -35.264389683}, {234., -35.264389683}, {252., -35.264389683}, {270., -35.264389683}, {288., -35.264389683}, {306., -35.264389683}, {324., -35.264389683}, {342., -35.264389683}, }; auto points = grid.to_points(); EXPECT(points.size() == grid.size()); ASSERT(points.size() == ref.size()); auto it = grid.begin(); for (size_t i = 0; i < points.size(); ++i) { EXPECT(points_equal(ref[i], points[i])); EXPECT(points_equal(ref[i], *it)); ++it; } EXPECT(it == grid.end()); size_t i = 0; for (const auto& it : grid) { EXPECT(points_equal(ref[i++], it)); } EXPECT(i == grid.size()); } CASE("crop") { spec::Custom a({{"grid", "o2"}}); std::unique_ptr grid1(GridFactory::build(a)); auto n1 = grid1->size(); EXPECT_EQUAL(n1, 88); a.set("south", 0.); std::unique_ptr grid2(GridFactory::build(a)); auto n2 = grid2->size(); EXPECT_EQUAL(n2, n1 / 2); spec::Custom b{{{"grid", "o2"}, {"west", -180}}}; std::unique_ptr grid3(GridFactory::build(b)); auto n3 = grid3->size(); EXPECT_EQUAL(n3, n1); EXPECT(grid3->boundingBox().periodic()); EXPECT(grid3->boundingBox().spec_str() == R"({"area":[90,-180,-90,180]})"); // (exclude Greenwhich meridian) std::unique_ptr grid4(grid3->make_grid_cropped(area::BoundingBox(90., -180., 0., -1.e-6))); auto n4 = grid4->size(); EXPECT_EQUAL(n4, n3 / 4); const std::vector ref{ {-180., 59.444408289}, {-162., 59.444408289}, {-144., 59.444408289}, {-126., 59.444408289}, {-108., 59.444408289}, {-90., 59.444408289}, {-72., 59.444408289}, {-54., 59.444408289}, {-36., 59.444408289}, {-18., 59.444408289}, {-180., 19.875719147}, {-165., 19.875719147}, {-150., 19.875719147}, {-135., 19.875719147}, {-120., 19.875719147}, {-105., 19.875719147}, {-90., 19.875719147}, {-75., 19.875719147}, {-60., 19.875719147}, {-45., 19.875719147}, {-30., 19.875719147}, {-15., 19.875719147}, }; auto points4 = grid4->to_points(); EXPECT(points4.size() == n4); ASSERT(points4.size() == ref.size()); auto it = grid4->begin(); for (size_t i = 0; i < points4.size(); ++i) { EXPECT(points_equal(ref[i], points4[i])); EXPECT(points_equal(ref[i], *it)); ++it; } EXPECT(it == grid4->end()); size_t i = 0; for (const auto& it : *grid4) { EXPECT(points_equal(ref[i++], it)); } EXPECT_EQUAL(i, n4); } CASE("equals") { std::unique_ptr grid1(GridFactory::build(spec::Custom({{"grid", "o3"}}))); std::unique_ptr grid2(GridFactory::make_from_string("N: 3")); std::unique_ptr grid3(new ReducedGaussian(3)); std::unique_ptr grid4(new ReducedGaussian(3, pl_type{20, 24, 28, 28, 24, 20})); EXPECT(*grid1 == *grid2); EXPECT(*grid2 == *grid3); EXPECT(*grid3 == *grid4); EXPECT(*grid4 == *grid1); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/range.cc0000664000175000017500000003204315161702250016323 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/geo/range/GaussianLatitude.h" #include "eckit/geo/range/Regular.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" #define EXPECT_APPROX(a, b) EXPECT(::eckit::types::is_approximately_equal((a), (b), 1e-3)) namespace eckit::geo::test { CASE("range::RegularXY") { SECTION("make_xy_range") { range::RegularXY range(1., -1., 2., -1.); EXPECT(range.size() == 4); EXPECT_APPROX(range.a(), -1.); EXPECT_APPROX(range.b(), 2.); EXPECT_APPROX(range.increment(), 1.); const auto& values = range.values(); EXPECT_APPROX(values[0], -1.); EXPECT_APPROX(values[1], 0.); EXPECT_APPROX(values[2], 1.); EXPECT_APPROX(values[3], 2.); } } CASE("range::RegularLongitude") { SECTION("1") { const range::RegularLongitude range(1., -1., 2.); EXPECT(!range.periodic()); EXPECT(range.size() == 4); EXPECT(range.a() == -1.); EXPECT(range.b() == 2.); const std::unique_ptr cropped(range.make_cropped_range(0.5, 359.5)); EXPECT(!cropped->periodic()); EXPECT(cropped->size() == 2); EXPECT(cropped->a() == 1.); EXPECT(cropped->b() == 2.); } SECTION("2") { const range::RegularLongitude range(1., 0.5, 359.5, 0.5); EXPECT(range.periodic()); EXPECT(range.size() == 360); const std::unique_ptr cropped1(range.make_cropped_range(-1., 2.)); EXPECT(!cropped1->periodic()); EXPECT(cropped1->size() == 3); } SECTION("3") { struct test { double inc; double a; double b; double ref; bool periodic; std::vector values; }; for (const auto& test : std::vector{ {1., -1., 2., -1., false, {-1, 0, 1, 2}}, {90., -90., 90., 0., false, {-90, 0, 90}}, {-90., 90., -90., 0., false, {90, 0, -90}}, {90., -180., 180., 0., true, {-180, -90, 0, 90}}, {45., -180., 180., 0., true, {-180, -135, -90, -45, 0, 45, 90, 135}}, }) { range::RegularLongitude range(test.inc, test.a, test.b, test.ref); EXPECT(range.periodic() == test.periodic); EXPECT(range.size() == test.values.size()); const auto& values = range.values(); EXPECT(values.size() == test.values.size()); for (auto a = test.values.begin(), b = values.begin(); a != test.values.end() && b != values.end(); ++a, ++b) { EXPECT_APPROX(*a, *b); } } } SECTION("4") { struct test { double inc; double a; double b; double ref; }; for (const auto& test : std::vector{ {1., 0., 360., 0.}, {1., 0., 360., 0.}, {1., 0.5, 359.5, 0.5}, {1., 0., 360., 0.5}, }) { range::RegularLongitude range(test.inc, test.a, test.b, test.ref); EXPECT(range.periodic()); EXPECT(range.size() == 360); } } SECTION("irregular") { for (const auto inc : {0.35, 0.7, 0.8, 1.4, 1.6}) { range::RegularLongitude range(inc, 0., 360.); EXPECT(range.periodic()); EXPECT(range.shift() == 0.); } } SECTION("degenerate") { range::RegularLongitude range1(1., 1., 1.); EXPECT(range1.size() == 1); EXPECT(range1.values().front() == 1.); range::RegularLongitude range2(2., 2., 2.); EXPECT(range2.size() == 1); EXPECT(range2.values().front() == 2.); range::RegularLongitude range3(0., 2., 2.); EXPECT(range3.size() == 1); EXPECT(range3.values().front() == 2.); range::RegularLongitude range4(0., 2., 2., 1.); EXPECT(range4.size() == 1); EXPECT(range4.values().front() == 2.); } SECTION("range [0, 360], cropped") { range::RegularLongitude range(10., 0., 360.); EXPECT(range.size() == 36); EXPECT(range.a() == 0.); EXPECT(range.b() == 360. - 10.); EXPECT(range.periodic()); const std::unique_ptr cropped1(range.make_cropped_range(-180., 180.)); EXPECT(cropped1->size() == 36); EXPECT(cropped1->a() == -180.); EXPECT(cropped1->b() == 180. - 10.); EXPECT(cropped1->periodic()); const std::unique_ptr cropped2(range.make_cropped_range(-180., 170.)); EXPECT(cropped2->size() == 36); EXPECT(cropped2->b() == 170.); EXPECT(cropped2->periodic()); const std::unique_ptr cropped3(range.make_cropped_range(-180., 160.)); EXPECT(cropped3->size() == 36 - 1); EXPECT(cropped3->b() == 160.); EXPECT_NOT(cropped3->periodic()); } SECTION("range [0, 180], cropped") { range::RegularLongitude range(10., 0., 180.); EXPECT(range.size() == 19); EXPECT_NOT(range.periodic()); const std::unique_ptr cropped1(range.make_cropped_range(1., 179.)); EXPECT(cropped1->size() == 19 - 2); EXPECT(cropped1->a() == 10.); EXPECT(cropped1->b() == 170.); EXPECT_NOT(cropped1->periodic()); const std::unique_ptr cropped2(range.make_cropped_range(1., 170.)); EXPECT(cropped2->size() == 19 - 2); EXPECT(cropped2->a() == 10.); EXPECT(cropped2->b() == 170.); EXPECT_NOT(cropped2->periodic()); const std::unique_ptr cropped3(range.make_cropped_range(-180., 180.)); EXPECT(cropped3->size() == 19); EXPECT(cropped3->a() == 0.); EXPECT(cropped3->b() == 180.); EXPECT_NOT(cropped3->periodic()); const std::unique_ptr cropped4(range.make_cropped_range(-190., 170.)); EXPECT(cropped4->size() == 19 - 1); EXPECT(cropped4->a() == 0.); EXPECT(cropped4->b() == 170.); EXPECT_NOT(cropped4->periodic()); } } CASE("range::RegularLatitude") { SECTION("simple") { range::RegularLatitude range1(1., -90., 90., 0.5); EXPECT(range1.size() == 180); EXPECT(range1.a() == -89.5); EXPECT(range1.b() == 89.5); range::RegularLatitude range2(-1., 90., -90., 0.5); EXPECT(range2.size() == 180); EXPECT(range2.a() == 89.5); EXPECT(range2.b() == -89.5); } SECTION("irregular") { for (const auto inc : {0.35, 0.7, 0.8, 1.4, 1.6}) { range::RegularLatitude range(inc, 0., 90.); EXPECT(range.shift() == 0.); } } } CASE("range::GaussianLatitude") { std::vector ref{59.44440828916676, 19.87571914744090, -19.87571914744090, -59.44440828916676}; SECTION("decreasing") { auto global = range::GaussianLatitude(2, false); EXPECT(global.includesNorthPole() && global.includesSouthPole()); EXPECT(global.size() == ref.size()); size_t i = 0; for (const auto& test : global.values()) { EXPECT_APPROX(test, ref[i++]); } std::unique_ptr cropped1(global.make_cropped_range(50., -50.)); EXPECT(cropped1->size() == 2); EXPECT_APPROX(cropped1->values()[0], ref[1]); EXPECT_APPROX(cropped1->values()[1], ref[2]); std::unique_ptr cropped2(global.make_cropped_range(59.445, -59.445)); std::unique_ptr cropped3(global.make_cropped_range(59.444, -59.444)); std::unique_ptr cropped4(global.make_cropped_range(59.444, -59.445)); EXPECT(cropped2->size() == 4); EXPECT(cropped3->size() == 2); EXPECT(cropped4->size() == 3); std::unique_ptr cropped5(global.make_cropped_range(-59.444, -59.445)); EXPECT(cropped5->size() == 1); EXPECT_APPROX(cropped5->values().front(), ref.back()); std::unique_ptr cropped6(global.make_cropped_range(90., 0.)); EXPECT(cropped6->size() == ref.size() / 2); EXPECT_APPROX(cropped6->values()[0], ref[0]); EXPECT_APPROX(cropped6->values()[1], ref[1]); auto crop_includes_poles = [&global](double north, double south) { std::unique_ptr cropped( dynamic_cast(global.make_cropped_range(north, south))); ASSERT(cropped); return std::make_pair(cropped->includesNorthPole(), cropped->includesSouthPole()); }; EXPECT(crop_includes_poles(70., -60.) == std::make_pair(true, true)); EXPECT(crop_includes_poles(60., -50.) == std::make_pair(true, false)); EXPECT(crop_includes_poles(50., -50.) == std::make_pair(false, false)); } SECTION("increasing") { std::vector rev(ref.rbegin(), ref.rend()); auto global = range::GaussianLatitude(2, true); EXPECT(global.size() == rev.size()); size_t i = 0; for (const auto& test : global.values()) { EXPECT_APPROX(test, rev[i++]); } std::unique_ptr cropped1(global.make_cropped_range(-50., 50.)); EXPECT(cropped1->size() == 2); EXPECT_APPROX(cropped1->values()[0], rev[1]); EXPECT_APPROX(cropped1->values()[1], rev[2]); std::unique_ptr cropped2(global.make_cropped_range(-59.445, 59.445)); std::unique_ptr cropped3(global.make_cropped_range(-59.444, 59.444)); std::unique_ptr cropped4(global.make_cropped_range(-59.445, 59.444)); EXPECT(cropped2->size() == 4); EXPECT(cropped3->size() == 2); EXPECT(cropped4->size() == 3); std::unique_ptr cropped5(global.make_cropped_range(-59.445, -59.444)); EXPECT(cropped5->size() == 1); EXPECT_APPROX(cropped5->values().front(), rev.front()); std::unique_ptr cropped6(global.make_cropped_range(0., 90.)); EXPECT(cropped6->size() == rev.size() / 2); EXPECT_APPROX(cropped6->values()[0], rev[2]); EXPECT_APPROX(cropped6->values()[1], rev[3]); auto crop_includes_poles = [&global](double north, double south) { std::unique_ptr cropped( dynamic_cast(global.make_cropped_range(south, north))); ASSERT(cropped); return std::make_pair(cropped->includesNorthPole(), cropped->includesSouthPole()); }; EXPECT(crop_includes_poles(70., -60.) == std::make_pair(true, true)); EXPECT(crop_includes_poles(60., -50.) == std::make_pair(true, false)); EXPECT(crop_includes_poles(50., -50.) == std::make_pair(false, false)); } } CASE("range::Regular") { struct test { double inc; size_t nlat; size_t nlon; }; for (const auto& test : std::vector{ {30., 7, 12}, // {10., 19, 36}, // {5., 37, 72}, // {2.5, 73, 144}, // {2., 91, 180}, // {1.8, 101, 200}, // {1.6, 113, 225}, // {1.5, 121, 240}, // {1.4, 129, 258}, // {1.25, 145, 288}, // {1.2, 151, 300}, // {1., 181, 360}, // {0.9, 201, 400}, // {0.8, 225, 450}, // {0.75, 241, 480}, // {0.7, 257, 515}, // {0.6, 301, 600}, // {0.5, 361, 720}, // {0.4, 451, 900}, // {0.35, 515, 1029}, // {0.3, 601, 1200}, // {0.25, 721, 1440}, // {0.2, 901, 1800}, // {0.15, 1201, 2400}, // {0.125, 1441, 2880}, // {0.1, 1801, 3600}, // {0.05, 3601, 7200}, // }) { range::RegularLatitude lat(test.inc, -90., 90., 0.); EXPECT_APPROX(lat.increment(), test.inc); EXPECT_EQUAL(lat.size(), test.nlat); range::RegularLongitude lon(test.inc, 0., 360., 0.); EXPECT_APPROX(lon.increment(), test.inc); EXPECT_EQUAL(lon.size(), test.nlon); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/search.cc0000664000175000017500000000563215161702250016500 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/container/KDTree.h" #include "eckit/geo/Grid.h" #include "eckit/geo/PointXY.h" #include "eckit/geo/PointXYZ.h" #include "eckit/geo/Search.h" #include "eckit/geo/projection/LonLatToXYZ.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { template struct Traits { using Point = PointT; using Payload = PayloadT; }; CASE("SearchXY (non-cacheable)") { using SearchXY = KDTreeMemory>; std::vector points{ {{0., 0.}, 0}, }; SearchXY search; search.build(points); auto a = search.nearestNeighbour({0.1, 0.}); EXPECT_EQUAL(a.payload(), 0); auto b = search.nearestNeighbour({0.9, 0.}); EXPECT_EQUAL(b.payload(), 0); search.insert({{1., 0.}, 1}); auto c = search.nearestNeighbour({0.1, 0.}); EXPECT_EQUAL(c.payload(), 0); auto d = search.nearestNeighbour({0.9, 0.}); EXPECT_EQUAL(d.payload(), 1); } CASE("SearchXYZ (non-cacheable)") { using SearchXYZ = KDTreeMemory>; std::vector points{ {{0., 0., 0.}, 0}, }; SearchXYZ search; search.build(points); auto a = search.nearestNeighbour({0.1, 0., 0.}); EXPECT_EQUAL(a.payload(), 0); auto b = search.nearestNeighbour({0.9, 0., 0.}); EXPECT_EQUAL(b.payload(), 0); search.insert({{1., 0., 0.}, 1}); auto c = search.nearestNeighbour({0.1, 0., 0.}); EXPECT_EQUAL(c.payload(), 0); auto d = search.nearestNeighbour({0.9, 0., 0.}); EXPECT_EQUAL(d.payload(), 1); } CASE("Search (cacheable)") { std::unique_ptr grid(GridFactory::build(spec::Custom{{{"grid", "30/30"}}})); const Search search(*grid); std::vector closest; projection::LonLatToXYZ to_xyz; Search::ValueType index = 0; auto points_equal = [](const auto& p, const auto& q) { return geo::points_equal(PointXYZ{p.x(0), p.x(1), p.x(2)}, PointXYZ{q.x(0), q.x(1), q.x(2)}); }; for (const auto& p : *grid) { Search::PointType q(to_xyz.fwd(std::get(p))); search.closestNPoints(q, 2, closest); EXPECT(closest.size() == 2); EXPECT(points_equal(closest.front().point(), q)); if (!std::get(p).pole()) { EXPECT(closest.front().payload() == index); } ++index; } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/projection_ll_to_xyz.cc0000664000175000017500000001100715161702250021503 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/geo/projection/LonLatToXYZ.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("projection: ll-to-xyz") { struct P : std::unique_ptr { explicit P(Projection* ptr) : unique_ptr(ptr) { ASSERT(unique_ptr::operator bool()); } }; struct test_t { PointLonLat a; PointXYZ b; }; constexpr double R = 1.; const auto L = R * std::sqrt(2.) / 2.; // spherical projections P to_xyz_1(ProjectionFactoryType::instance().get("ll-to-xyz").create(spec::Custom{{"R", 1.}})); P to_xyz_2(new projection::LonLatToXYZ(1., 1.)); EXPECT(*to_xyz_1 == *to_xyz_2); EXPECT(*to_xyz_1 == projection::LonLatToXYZ(1.)); // oblate spheroid projections P to_xyz_3(ProjectionFactoryType::instance().get("ll-to-xyz").create(spec::Custom{{"a", 1.}, {"b", 0.5}})); P to_xyz_4(new projection::LonLatToXYZ(1., 0.5)); EXPECT(*to_xyz_3 == *to_xyz_4); // problate spheroid (not supported) EXPECT_THROWS(projection::LonLatToXYZ(0.5, 1.)); SECTION("spec") { Log::info() << to_xyz_1->spec_str() << std::endl; Log::info() << to_xyz_2->spec_str() << std::endl; Log::info() << to_xyz_3->spec_str() << std::endl; Log::info() << to_xyz_4->spec_str() << std::endl; EXPECT(to_xyz_1->spec_str() == R"({"r":1,"type":"ll-to-xyz"})"); EXPECT(to_xyz_2->spec_str() == R"({"r":1,"type":"ll-to-xyz"})"); EXPECT(to_xyz_3->spec_str() == R"({"a":1,"b":0.5,"type":"ll-to-xyz"})"); EXPECT(to_xyz_4->spec_str() == R"({"a":1,"b":0.5,"type":"ll-to-xyz"})"); } SECTION("roundtrip") { for (const auto& p : { PointLonLat{1, 1}, PointLonLat{1, 0}, PointLonLat(723., 1.), }) { EXPECT(points_equal(p, to_xyz_1->inv(to_xyz_1->fwd(p)))); EXPECT(points_equal(p, to_xyz_2->inv(to_xyz_2->fwd(p)))); EXPECT(points_equal(to_xyz_1->fwd(p), to_xyz_2->fwd(p))); EXPECT(points_equal(to_xyz_2->fwd(p), to_xyz_1->fwd(p))); if (p.lat() == 0) { auto q = to_xyz_3->fwd(p); EXPECT(points_equal(to_xyz_1->fwd(p), q)); EXPECT(points_equal(to_xyz_2->inv(q), p)); } } } SECTION("sphere (ll -> xyz, xyz -> ll)") { for (const auto& test : std::vector{ {{0, 90}, {0, 0, R}}, // {{0, -90}, {0, 0, -R}}, // {{0, 0}, {R, 0, 0}}, // {{-360, 0}, {R, 0, 0}}, // {{90, 0}, {0, R, 0}}, // {{-270, 0}, {0, R, 0}}, // {{180, 0}, {-R, 0, 0}}, // {{-180, 0}, {-R, 0, 0}}, // {{270, 0}, {0, -R, 0}}, // {{-90, 0}, {0, -R, 0}}, // {{45, 0}, {L, L, 0}}, // {{-315, 0}, {L, L, 0}}, // {{135, 0}, {-L, L, 0}}, // {{-225, 0}, {-L, L, 0}}, // {{225, 0}, {-L, -L, 0}}, // {{-135, 0}, {-L, -L, 0}}, // {{315, 0}, {L, -L, 0}}, // {{-45, 0}, {L, -L, 0}}, // }) { EXPECT(points_equal(to_xyz_1->fwd(test.a), test.b)); EXPECT(points_equal(to_xyz_1->inv(test.b), test.a)); EXPECT(points_equal(to_xyz_2->fwd(test.a), test.b)); EXPECT(points_equal(to_xyz_2->inv(test.b), test.a)); } } SECTION("spheroid (ll -> xyz)") { for (const auto& test : std::vector{ {{0, -90}, {0, 0, -0.5}}, // {{42, -90}, {0, 0, -0.5}}, // {{0, 90}, {0, 0, 0.5}}, // {{42, 90}, {0, 0, 0.5}}, // }) { EXPECT(points_equal(to_xyz_3->fwd(test.a), test.b)); EXPECT(points_equal(to_xyz_4->fwd(test.a), test.b)); } } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/grid_reduced_ll.cc0000664000175000017500000003631115161702250020340 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/geo/Grid.h" #include "eckit/geo/Point.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("points (global)") { spec::Custom spec{{"type", "reduced_ll"}, {"pl", std::vector{20, 24, 24, 20}}}; std::unique_ptr grid(GridFactory::build(spec)); EXPECT_EQUAL(grid->size(), 88); const std::vector ref{ {0., 90.}, {18., 90.}, {36., 90.}, {54., 90.}, {72., 90.}, // {90., 90.}, {108., 90.}, {126., 90.}, {144., 90.}, {162., 90.}, // {180., 90.}, {198., 90.}, {216., 90.}, {234., 90.}, {252., 90.}, // {270., 90.}, {288., 90.}, {306., 90.}, {324., 90.}, {342., 90.}, // {0., 30.}, {15., 30.}, {30., 30.}, {45., 30.}, {60., 30.}, // {75., 30.}, {90., 30.}, {105., 30.}, {120., 30.}, {135., 30.}, // {150., 30.}, {165., 30.}, {180., 30.}, {195., 30.}, {210., 30.}, // {225., 30.}, {240., 30.}, {255., 30.}, {270., 30.}, {285., 30.}, // {300., 30.}, {315., 30.}, {330., 30.}, {345., 30.}, // {0., -30.}, {15., -30.}, {30., -30.}, {45., -30.}, {60., -30.}, // {75., -30.}, {90., -30.}, {105., -30.}, {120., -30.}, {135., -30.}, // {150., -30.}, {165., -30.}, {180., -30.}, {195., -30.}, {210., -30.}, // {225., -30.}, {240., -30.}, {255., -30.}, {270., -30.}, {285., -30.}, // {300., -30.}, {315., -30.}, {330., -30.}, {345., -30.}, // {0., -90.}, {18., -90.}, {36., -90.}, {54., -90.}, {72., -90.}, // {90., -90.}, {108., -90.}, {126., -90.}, {144., -90.}, {162., -90.}, // {180., -90.}, {198., -90.}, {216., -90.}, {234., -90.}, {252., -90.}, {270., -90.}, {288., -90.}, {306., -90.}, {324., -90.}, {342., -90.}, }; auto points = grid->to_points(); EXPECT(points.size() == grid->size()); ASSERT(points.size() == ref.size()); auto it = grid->begin(); for (size_t i = 0; i < points.size(); ++i) { EXPECT(points_equal(ref[i], points[i])); EXPECT(points_equal(ref[i], *it)); ++it; } EXPECT(it == grid->end()); size_t i = 0; for (const auto& it : *grid) { EXPECT(points_equal(ref[i++], it)); } EXPECT(i == grid->size()); } CASE("points (regional)") { spec::Custom spec{ {"type", "reduced_ll"}, // {"pl", std::vector{ 2, 4, 6, 8, 12, 14, 16, 20, 22, 24, 28, 30, 32, 36, 38, 40, 42, 46, 48, 50, 54, 56, 58, 62, 64, 66, 70, 72, 74, 78, 80, 82, 86, 88, 90, 94, 96, 98, 102, 104, 106, 110, 112, 114, 118, 120, 122, 126, 128, 130, 134, 136, 138, 140, 144, 146, 148, 152, 154, 156, 160, 162, 164, 168, 170, 172, 176, 178, 180, 184, 186, 188, 192, 194, 196, 200, 202, 204, 206, 210, 212, 214, 218, 220, 222, 226, 228, 230, 234, 236, 238, 242, 244, 246, 248, 252, 254, 256, 260, 262, 264, 268, 270, 272, 276, 278, 280, 282, 286, 288, 290, 294, 296, 298, 302, 304, 306, 308, 312, 314, 316, 320, 322, 324, 328, 330, 332, 334, 338, 340, 342, 346, 348, 350, 352, 356, 358, 360, 364, 366, 368, 372, 374, 376, 378, 382, 384, 386, 390, 392, 394, 396, 400, 402, 404, 406, 410, 412, 414, 418, 420, 422, 424, 428, 430, 432, 436, 438, 440, 442, 446, 448, 450, 452, 456, 458, 460, 462, 466, 468, 470, 474, 476, 478, 480, 484, 486, 488, 490, 494, 496, 498, 500, 504, 506, 508, 510, 514, 516, 518, 520, 524, 526, 528, 530, 534, 536, 538, 540, 544, 546, 548, 550, 554, 556, 558, 560, 562, 566, 568, 570, 572, 576, 578, 580, 582, 586, 588, 590, 592, 594, 598, 600, 602, 604, 606, 610, 612, 614, 616, 620, 622, 624, 626, 628, 632, 634, 636, 638, 640, 644, 646, 648, 650, 652, 656, 658, 660, 662, 664, 668, 670, 672, 674, 676, 680, 682, 684, 686, 688, 692, 694, 696, 698, 700, 702, 706, 708, 710, 712, 714, 716, 720, 722, 724, 726, 728, 730, 734, 736, 738, 740, 742, 744, 748, 750, 752, 754, 756, 758, 762, 764, 766, 768, 770, 772, 774, 778, 780, 782, 784, 786, 788, 790, 792, 796, 798, 800, 802, 804, 806, 808, 812, 814, 816, 818, 820, 822, 824, 826, 828, 832, 834, 836, 838, 840, 842, 844, 846, 848, 852, 854, 856, 858, 860, 862, 864, 866, 868, 870, 872, 876, 878, 880, 882, 884, 886, 888, 890, 892, 894, 896, 898, 900, 904, 906, 908, 910, 912, 914, 916, 918, 920, 922, 924, 926, 928, 930, 932, 934, 936, 940, 942, 944, 946, 948, 950, 952, 954, 956, 958, 960, 962, 964, 966, 968, 970, 972, 974, 976, 978, 980, 982, 984, 986, 988, 990, 992, 994, 996, 998, 1000, 1002, 1004, 1006, 1008, 1010, 1012, 1014, 1016, 1018, 1020, 1022, 1024, 1026, 1028, 1030, 1032, 1034, 1036, 1038, 1040, 1042, 1044, 1046, 1048, 1050, 1052, 1054, 1056, 1058, 1058, 1060, 1062, 1064, 1066, 1068, 1070, 1072, 1074, 1076, 1078, 1080, 1082, 1084, 1086, 1088, 1090, 1090, 1092, 1094, 1096, 1098, 1100, 1102, 1104, 1106, 1108, 1110, 1112, 1112, 1114, 1116, 1118, 1120, 1122, 1124, 1126, 1128, 1130, 1130, 1132, 1134, 1136, 1138, 1140, 1142, 1144, 1144, 1146, 1148, 1150, 1152, 1154, 1156, 1158, 1158, 1160, 1162, 1164, 1166, 1168, 1170, 1170, 1172, 1174, 1176, 1178, 1180, 1180, 1182, 1184, 1186, 1188, 1190, 1190, 1192, 1194, 1196, 1198, 1200, 1200, 1202, 1204, 1206, 1208, 1208, 1210, 1212, 1214, 1216, 1216, 1218, 1220, 1222, 1224, 1224, 1226, 1228, 1230, 1232, 1232, 1234, 1236, 1238, 1238, 1240, 1242, 1244, 1244, 1246, 1248, 1250, 1250, 1252, 1254, 1256, 1256, 1258, 1260, 1262, 1262, 1264, 1266, 1268, 1268, 1270, 1272, 1274, 1274, 1276, 1278, 1280, 1280, 1282, 1284, 1284, 1286, 1288, 1288, 1290, 1292, 1294, 1294, 1296, 1298, 1298, 1300, 1302, 1302, 1304, 1306, 1306, 1308, 1310, 1312, 1312, 1314, 1316, 1316, 1318, 1320, 1320, 1322, 1324, 1324, 1326, 1326, 1328, 1330, 1330, 1332, 1334, 1334, 1336, 1338, 1338, 1340, 1340, 1342, 1344, 1344, 1346, 1348, 1348, 1350, 1350, 1352, 1354, 1354, 1356, 1356, 1358, 1360, 1360, 1362, 1362, 1364, 1366, 1366, 1368, 1368, 1370, 1372, 1372, 1374, 1374, 1376, 1376, 1378, 1378, 1380, 1382, 1382, 1384, 1384, 1386, 1386, 1388, 1388, 1390, 1392, 1392, 1394, 1394, 1396, 1396, 1398, 1398, 1400, 1400, 1402, 1402, 1404, 1404, 1406, 1406, 1408, 1408, 1410, 1410, 1412, 1412, 1414, 1414, 1416, 1416, 1418, 1418, 1420, 1420, 1422, 1422, 1424, 1424, 1426, 1426, 1428, 1428, 1428, 1430, 1430, 1432, 1432, 1434, 1434, 1436, 1436, 1436, 1438, 1438, 1440, 1440, 1442, 1442, 1442, 1444, 1444, 1446, 1446, 1448, 1448, 1448, 1450, 1450, 1452, 1452, 1452, 1454, 1454, 1456, 1456, 1456, 1458, 1458, 1458, 1460, 1460, 1462, 1462, 1462, 1464, 1464, 1464, 1466, 1466, 1466, 1468, 1468, 1468, 1470, 1470, 1472, 1472, 1472, 1474, 1474, 1474, 1476, 1476, 1476, 1476, 1478, 1478, 1478, 1480, 1480, 1480, 1482, 1482, 1482, 1484, 1484, 1484, 1484, 1486, 1486, 1486, 1488, 1488, 1488, 1488, 1490, 1490, 1490, 1490, 1492, 1492, 1492, 1494, 1494, 1494, 1494, 1496, 1496, 1496, 1496, 1496, 1498, 1498, 1498, 1498, 1500, 1500, 1500, 1500, 1502, 1502, 1502, 1502, 1502, 1504, 1504, 1504, 1504, 1504, 1506, 1506, 1506, 1506, 1506, 1508, 1508, 1508, 1508, 1508, 1508, 1510, 1510, 1510, 1510, 1510, 1510, 1512, 1512, 1512, 1512, 1512, 1512, 1512, 1514, 1514, 1514, 1514, 1514, 1514, 1514, 1514, 1516, 1516, 1516, }}, // {"area", std::vector{90., -98., 5., 54.}}, // }; const std::vector ref{ {-98., 90.}, {54., 90.}, {-98., 89.9}, {-47.333333333, 89.9}, {3.333333333, 89.9}, {54., 89.9}, {-98., 89.8}, {-67.6, 89.8}, {-37.2, 89.8}, {-6.8, 89.8}, {23.6, 89.8}, {54., 89.8}, {-98., 89.7}, {-76.285714286, 89.7}, {-54.571428571, 89.7}, {-32.857142857, 89.7}, {-11.142857143, 89.7}, {10.571428571, 89.7}, {32.285714286, 89.7}, {54., 89.7}, {-98., 89.6}, {-84.181818182, 89.6}, {-70.363636364, 89.6}, {-56.545454545, 89.6}, {-42.727272727, 89.6}, {-28.909090909, 89.6}, {-15.090909091, 89.6}, {-1.272727273, 89.6}, {12.545454545, 89.6}, {26.363636364, 89.6}, {40.181818182, 89.6}, {54., 89.6}, {-98., 89.5}, {-86.307692308, 89.5}, {-74.615384615, 89.5}, {-62.923076923, 89.5}, {-51.230769231, 89.5}, {-39.538461538, 89.5}, {-27.846153846, 89.5}, {-16.153846154, 89.5}, {-4.461538462, 89.5}, {7.230769231, 89.5}, {18.923076923, 89.5}, {30.615384615, 89.5}, {42.307692308, 89.5}, {54., 89.5}, {-98., 89.4}, {-87.866666667, 89.4}, {-77.733333333, 89.4}, {-67.6, 89.4}, {-57.466666667, 89.4}, {-47.333333333, 89.4}, {-37.2, 89.4}, {-27.066666667, 89.4}, {-16.933333333, 89.4}, {-6.8, 89.4}, {3.333333333, 89.4}, {13.466666667, 89.4}, {23.6, 89.4}, {33.733333333, 89.4}, {43.866666667, 89.4}, {54., 89.4}, {-98., 89.3}, {-90., 89.3}, {-82., 89.3}, {-74., 89.3}, {-66., 89.3}, {-58., 89.3}, {-50., 89.3}, }; constexpr size_t EXPECTED_SIZE = 796702; std::unique_ptr grid(GridFactory::build(spec)); auto points = grid->to_points(); EXPECT(grid->size() == EXPECTED_SIZE); EXPECT(points.size() == EXPECTED_SIZE); auto it = grid->begin(); for (size_t i = 0; i < ref.size(); ++i) { EXPECT(points_equal(ref[i], points[i])); EXPECT(points_equal(ref[i], *it)); ++it; } } CASE("points (contains zeros)") { spec::Custom spec{ {"type", "reduced_ll"}, // {"pl", std::vector{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 164, 170, 176, 182, 188, 194, 200, 206, 212, 218, 224, 230, 236, 244, 250, 256, 262, 268, 274, 280, 286, 292, 298, 304, 310, 316, 322, 328, 334, 340, 346, 352, 356, 362, 368, 374, 380, 386, 392, 398, 404, 410, 414, 420, 426, 432, 438, 444, 448, 454, 460, 466, 472, 476, 482, 488, 494, 498, 504, 510, 514, 520, 526, 532, 536, 542, 546, 552, 558, 562, 568, 572, 578, 584, 588, 594, 598, 604, 608, 614, 618, 624, 628, 634, 638, 642, 648, 652, 658, 662, 666, 672, 676, 680, 686, 690, 694, 698, 704, 708, 712, 716, 720, 726, 730, 734, 738, 742, 746, 750, 754, 758, 762, 766, 772, 776, 778, 782, 786, 790, 794, 798, 802, 806, 810, 814, 816, 820, 824, 828, 832, 834, 838, 842, 844, 848, 852, 854, 858, 862, 864, 868, 870, 874, 876, 880, 882, 886, 888, 892, 894, 898, 900, 902, 906, 908, 910, 914, 916, 918, 920, 924, 926, 928, 930, 932, 934, 938, 940, 942, 944, 946, 948, 950, 952, 954, 956, 958, 960, 960, 962, 964, 966, 968, 970, 970, 972, 974, 976, 976, 978, 980, 980, 982, 982, 984, 986, 986, 988, 988, 990, 990, 990, 992, 992, 994, 994, 994, 996, 996, 996, 998, 998, 998, 998, 998, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 998, 998, 998, 998, 998, 996, 996, 996, 994, 994, 994, 992, 992, 990, 990, 990, 988, 988, 986, 986, 984, 982, 982, 980, 980, 978, 976, 976, 974, 972, 970, 970, 968, 966, 964, 962, 960, 960, 958, 956, 954, 952, 950, 948, 946, 944, 942, 940, 938, 934, 932, 930, 928, 926, 924, 920, 918, 916, 914, 910, 908, 906, 902, 900, 898, 894, 892, 888, 886, 882, 880, 876, 874, 870, 868, 864, 862, 858, 854, 852, 848, 844, 842, 838, 834, 832, 828, 824, 820, 816, 814, 810, 806, 802, 798, 794, 790, 786, 782, 778, 776, 772, 766, 762, 758, 754, 750, 746, 742, 738, 734, 730, 726, 720, 716, 712, 708, 704, 698, 694, 690, 686, 680, 676, 672, 666, 662, 658, 652, 648, 642, 638, 634, 628, 624, 618, 614, 608, 604, 598, 594, 588, 584, 578, 572, 568, 562, 558, 552, 546, 542, 536, 532, 526, 520, 514, 510, 504, 498, 494, 488, 482, 476, 472, 466, 460, 454, 448, 444, 438, 432, 426, 420, 414, 410, 404, 398, 392, 386, 380, 374, 368, 362, 356, 352, 346, 340, 334, 328, 322, 316, 310, 304, 298, 292, 286, 280, 274, 268, 262, 256, 250, 244, 236, 230, 224, 218, 212, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, {"area", std::vector{90., 0., -90., 359.64}}, }; constexpr size_t EXPECTED_SIZE = 313362; std::unique_ptr grid(GridFactory::build(spec)); auto points = grid->to_points(); EXPECT(grid->boundingBox().global()); EXPECT(grid->size() == EXPECTED_SIZE); EXPECT(points.size() == EXPECTED_SIZE); EXPECT(points_equal(points[0], PointLonLat{0., 81.})); EXPECT(points_equal(points[1], PointLonLat{2.3076923077, 81.})); EXPECT(points_equal(points[78], PointLonLat{180., 81.})); EXPECT(points_equal(points[79], PointLonLat{182.3076923077, 81.})); EXPECT(points_equal(points[156], PointLonLat{0., 80.64})); EXPECT(points_equal(points[157], PointLonLat{2.1951219512, 80.64})); EXPECT(points_equal(points[238], PointLonLat{180., 80.64})); EXPECT(points_equal(points[239], PointLonLat{182.1951219512, 80.64})); EXPECT(points_equal(points[320], PointLonLat{0., 80.28})); EXPECT(points_equal(points[321], PointLonLat{2.1176470588, 80.28})); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/grid.cc0000664000175000017500000000567115161702250016163 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/geo/Grid.h" #include "eckit/geo/area/BoundingBox.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("Grid from grid_spec") { SECTION("GridSpec canonical") { struct { const char* grid; const char* canonical; size_t size; } tests[]{ {"{grid: [10, 10]}", R"({"grid":[10,10]})", 684}, // {"{grid: [20, 10]}", R"({"grid":[20,10]})", 342}, // {"{pl: [20, 24, 24, 20]}", R"({"grid":"O2"})", 88}, // {"{grid: o8}", R"({"grid":"O8"})", 544}, // {"{grid: h2_ring}", R"({"grid":"H2"})", 48}, // {"{grid: h2n}", R"({"grid":"H2","order":"nested"})", 48}, // }; for (const auto& test : tests) { std::unique_ptr grid(GridFactory::make_from_string(test.grid)); EXPECT(grid->size() == test.size); EXPECT(grid->spec_str() == test.canonical); static const auto bbox_spec_str = area::BoundingBox::bounding_box_default().spec_str(); EXPECT(grid->boundingBox().spec_str() == bbox_spec_str); } } } CASE("Grid from name") { SECTION("GridFactory::build_from_name") { struct { const char* name; size_t size; } tests[]{{"O2", 88}, {"f2", 32}, {"h2", 48}}; for (const auto& test : tests) { std::unique_ptr a(GridFactory::build(spec::Custom({{"grid", test.name}}))); EXPECT_EQUAL(test.size, a->size()); std::unique_ptr b(GridFactory::make_from_string(test.name)); EXPECT_EQUAL(test.size, b->size()); } } } CASE("Grid from increments") { SECTION("global") { std::unique_ptr global(GridFactory::build(spec::Custom({ {"type", "regular_ll"}, {"grid", std::vector{1, 1}}, }))); EXPECT_EQUAL(global->size(), 360 * 181); } SECTION("non-global") { std::unique_ptr grid(GridFactory::build(spec::Custom({ {"type", "regular_ll"}, {"grid", std::vector{1, 1}}, {"north", 10}, {"west", 1}, {"south", 1}, {"east", 10}, }))); EXPECT_EQUAL(grid->size(), 100); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/point.cc0000664000175000017500000000311315161702250016354 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/geo/Point.h" #include "eckit/geo/PointLonLat.h" #include "eckit/geo/PointXY.h" #include "eckit/geo/PointXYZ.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("Point comparison") { auto r(PointLonLat::make(-10., -91.)); EXPECT(points_equal(r, r.antipode().antipode())); for (Point a1 : {PointLonLat{-180., 0.}, PointLonLat{180., 10.}, PointLonLat{0., 90.}, PointLonLat{1., -90.}}) { auto a2 = PointLonLat::make(std::get(a1).lon() + 720., std::get(a1).lat()); EXPECT(points_equal(a1, a2)); } PointXY p2{1., 2.}; PointXYZ p3{1., 2., 3.}; PointLonLat pll{4., 5.}; for (const auto& ab : std::vector>{{p2, p2}, {p3, p3}, {pll, pll}}) { EXPECT(points_equal(ab.first, ab.second)); } for (const auto& ab : std::vector>{{p2, p3}, {p2, pll}, {p3, p2}, {p3, pll}, {pll, p2}, {pll, p3}}) { EXPECT_THROWS_AS(points_equal(ab.first, ab.second), AssertionFailed); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/projection.cc0000664000175000017500000000321715161702250017404 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/Projection.h" #include "eckit/geo/figure/UnitSphere.h" #include "eckit/geo/projection/LonLatToXYZ.h" // to test Reverse #include "eckit/geo/projection/Reverse.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("projection: none") { Point p = PointLonLat{1, 1}; std::unique_ptr projection(ProjectionFactoryType::instance().get("none").create(spec::Custom{})); EXPECT(points_equal(p, projection->inv(p))); EXPECT(points_equal(p, projection->fwd(p))); } CASE("projection: reverse") { projection::LonLatToXYZ ab(new figure::UnitSphere); projection::Reverse ba(new figure::UnitSphere); PointLonLat p = NORTH_POLE; PointXYZ q{0., 0., 1.}; ASSERT(points_equal(q, ab.fwd(p))); ASSERT(points_equal(p, ab.inv(q))); // ensure fwd(PointXYZ) -> PointLonLat, inv(PointLonLat) -> PointXYZ EXPECT(points_equal(p, ba.fwd(q))); EXPECT(points_equal(q, ba.inv(p))); ASSERT(ab.spec().get_string("type") == "ll-to-xyz"); EXPECT(ba.spec().get_string("type") == "reverse-ll-to-xyz"); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/figure.cc0000664000175000017500000001770615161702250016521 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/geo/Figure.h" #include "eckit/geo/PointLonLat.h" #include "eckit/geo/PointXYZ.h" #include "eckit/geo/area/BoundingBox.h" #include "eckit/geo/figure/OblateSpheroid.h" #include "eckit/geo/figure/Sphere.h" #include "eckit/geo/figure/SphereT.h" #include "eckit/geo/figure/UnitSphere.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::test { struct F : std::unique_ptr
{ explicit F(Figure* ptr) : unique_ptr(ptr) { ASSERT(unique_ptr::operator bool()); } }; struct DatumTwoUnits { static constexpr double radius = 2.; static constexpr double a = radius; static constexpr double b = radius; }; CASE("Sphere") { F f1(FigureFactory::build(spec::Custom{{"R", 1.}})); F f2(FigureFactory::build(spec::Custom{{"a", 1.}, {"b", 1.}})); F f3(new figure::Sphere(1.)); EXPECT_THROWS_AS(figure::Sphere(-1.), BadValue); EXPECT(*f1 == *f2); EXPECT(*f1 == *f3); auto e = f1->eccentricity(); EXPECT(types::is_approximately_equal(e, 0.)); EXPECT(f1->spec_str() == R"({"r":1})"); } CASE("Oblate spheroid") { F f1(FigureFactory::build(spec::Custom{{"b", 0.5}, {"a", 1.}})); F f2(new figure::OblateSpheroid(1., 0.5)); EXPECT_THROWS_AS(figure::OblateSpheroid(0.5, 1.), BadValue); // prolate spheroid EXPECT_THROWS_AS(figure::OblateSpheroid(1., -1.), BadValue); EXPECT(*f1 == *f2); auto e = f1->eccentricity(); EXPECT(types::is_strictly_greater(e, 0.)); EXPECT(f1->spec_str() == R"({"a":1,"b":0.5})"); } CASE("Unit Sphere") { figure::UnitSphere s1; const auto R = s1.radius(); const auto L = R * std::sqrt(2) / 2.; const PointLonLat P1(-71.6, -33.); // Valparaíso const PointLonLat P2(121.8, 31.4); // Shanghai SECTION("radius") { EXPECT(s1.radius() == 1.); } SECTION("north pole") { EXPECT(points_equal(s1._convertSphericalToCartesian({0., 90.}), PointXYZ{0, 0, R})); } SECTION("south pole") { EXPECT(points_equal(s1._convertSphericalToCartesian({0., -90.}), PointXYZ{0, 0, -R})); } SECTION("distances") { // Same points with added shifts auto P1b = PointLonLat::make(288.4, -33.); // Valparaíso + longitude shift auto P2b = PointLonLat::make(301.8, 148.6); // Shanghai + latitude/longitude shift auto P2c = PointLonLat::make(-58.2, -211.4); // Shanghai + latitude/longitude shift auto d0 = s1.distance(P1, P2); auto d1 = s1.distance(P1b, P2); auto d2 = s1.distance(P1, P2b); auto d3 = s1.distance(P1, P2c); EXPECT(types::is_approximately_equal(d0, d1)); EXPECT(types::is_approximately_equal(d0, d2)); EXPECT(types::is_approximately_equal(d0, d3)); } SECTION("area globe") { EXPECT(s1.area() == 4. * M_PI * R * R); } SECTION("area hemispheres") { auto area_hemisphere_north = s1._area({90., -180., 0., 180.}); auto area_hemisphere_south = s1._area({0., -180., -90., 180.}); EXPECT(area_hemisphere_north == 0.5 * s1.area()); EXPECT(area_hemisphere_north == area_hemisphere_south); } SECTION("lon 0 (quadrant)") { EXPECT(points_equal(s1._convertSphericalToCartesian({0., 0.}), PointXYZ{R, 0, 0})); EXPECT(points_equal(s1._convertSphericalToCartesian({-360., 0.}), PointXYZ{R, 0, 0})); } SECTION("lon 90 (quadrant)") { EXPECT(points_equal(s1._convertSphericalToCartesian({90., 0.}), PointXYZ{0, R, 0})); EXPECT(points_equal(s1._convertSphericalToCartesian({-270., 0.}), PointXYZ{0, R, 0})); } SECTION("lon 180 (quadrant)") { EXPECT(points_equal(s1._convertSphericalToCartesian({180., 0.}), PointXYZ{-R, 0, 0})); EXPECT(points_equal(s1._convertSphericalToCartesian({-180., 0.}), PointXYZ{-R, 0, 0})); } SECTION("lon 270 (quadrant)") { EXPECT(points_equal(s1._convertSphericalToCartesian({270., 0.}), PointXYZ{0, -R, 0})); EXPECT(points_equal(s1._convertSphericalToCartesian({-90., 0.}), PointXYZ{0, -R, 0})); } SECTION("lon 45 (octant)") { EXPECT(points_equal(s1._convertSphericalToCartesian({45., 0.}), PointXYZ{L, L, 0})); EXPECT(points_equal(s1._convertSphericalToCartesian({-315., 0.}), PointXYZ{L, L, 0})); } SECTION("lon 135 (octant)") { EXPECT(points_equal(s1._convertSphericalToCartesian({135., 0.}), PointXYZ{-L, L, 0})); EXPECT(points_equal(s1._convertSphericalToCartesian({-225., 0.}), PointXYZ{-L, L, 0})); } SECTION("lon 225 (octant)") { EXPECT(points_equal(s1._convertSphericalToCartesian({225., 0.}), PointXYZ{-L, -L, 0})); EXPECT(points_equal(s1._convertSphericalToCartesian({-135., 0.}), PointXYZ{-L, -L, 0})); } SECTION("lon 315 (octant)") { EXPECT(points_equal(s1._convertSphericalToCartesian({315., 0.}), PointXYZ{L, -L, 0})); EXPECT(points_equal(s1._convertSphericalToCartesian({-45., 0.}), PointXYZ{L, -L, 0})); } SECTION("lat 100") { // Default behavior throws EXPECT_THROWS_AS(PointLonLat::assert_latitude_range(PointLonLat(0., 100.)), BadValue); auto p = s1._convertSphericalToCartesian(PointLonLat::make(0., 100.), 0.); auto q = s1._convertSphericalToCartesian(PointLonLat::make(180., 80.), 0.); // sin(x) and sin(pi-x) are not bitwise identical EXPECT(types::is_approximately_equal(p.X(), q.X())); EXPECT(types::is_approximately_equal(p.Y(), q.Y())); EXPECT(types::is_approximately_equal(p.Z(), q.Z())); } SECTION("lat 290") { // Default behavior throws EXPECT_THROWS_AS(PointLonLat::assert_latitude_range(PointLonLat(15., 290.)), BadValue); auto p = s1._convertSphericalToCartesian(PointLonLat::make(15., 290.), 0.); auto q = s1._convertSphericalToCartesian(PointLonLat::make(15., -70.), 0.); // sin(x) and sin(pi-x) are not bitwise identical EXPECT(types::is_approximately_equal(p.X(), q.X())); EXPECT(types::is_approximately_equal(p.Y(), q.Y())); EXPECT(types::is_approximately_equal(p.Z(), q.Z())); } SECTION("lat -120") { // Default behavior throws EXPECT_THROWS_AS(PointLonLat::assert_latitude_range(PointLonLat(45., -120.)), BadValue); auto p = s1._convertSphericalToCartesian(PointLonLat::make(45., -120.), 0.); auto q = s1._convertSphericalToCartesian(PointLonLat::make(225., -60.), 0.); // sin(x) and sin(pi-x) are not bitwise identical EXPECT(types::is_approximately_equal(p.X(), q.X())); EXPECT(types::is_approximately_equal(p.Y(), q.Y())); EXPECT(types::is_approximately_equal(p.Z(), q.Z())); } } CASE("Two-unit Sphere") { figure::UnitSphere s1; figure::SphereT s2; const PointLonLat P1(-71.6, -33.); // Valparaíso const PointLonLat P2(121.8, 31.4); // Shanghai SECTION("radius") { EXPECT(s2.radius() == 2.); } SECTION("distances") { auto distance_1 = s1.distance(P1, P2); auto distance_2 = s2.distance(P1, P2); EXPECT(2. * distance_1 == distance_2); } SECTION("area") { auto global_1 = s1.area(); auto global_2 = s2.area(); EXPECT(4. * global_1 == global_2); area::BoundingBox bbox({P2.lat(), P1.lon(), P1.lat(), P2.lon()}); auto local_1 = s1._area(bbox); auto local_2 = s2._area(bbox); EXPECT(4. * local_1 == local_2); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/eckit_geo_cache/0000775000175000017500000000000015161702250017772 5ustar alastairalastaireckit-2.0.7/tests/geo/eckit_geo_cache/grid/0000775000175000017500000000000015161702250020717 5ustar alastairalastaireckit-2.0.7/tests/geo/eckit_geo_cache/grid/orca/0000775000175000017500000000000015161702250021643 5ustar alastairalastair././@LongLink0000644000000000000000000000016500000000000011605 Lustar rootrooteckit-2.0.7/tests/geo/eckit_geo_cache/grid/orca/d5bde4f52ff3a9bea5629cd9ac514410-742e1f23882c68571cd820ecc3edce10.ekeckit-2.0.7/tests/geo/eckit_geo_cache/grid/orca/d5bde4f52ff3a9bea5629cd9ac514410-742e1f23882c68571cd0000664000175000017500000030764515161702250031005 0ustar alastairalastairATLAS-IO 3L` yamlxxh64:0def9db1b524f6c5  METADATA-BEGIN { "version" : { "type" : "scalar", "datatype" : "int32", "value" : 0, "base64" : "AAAAAA==" }, "dimensions" : { "type" : "array", "shape" : [2], "datatype" : "int32", "value" : [182,149], "data" : { "section" : 1, "compression" : { "type" : "lz4" } } }, "halo" : { "type" : "array", "shape" : [4], "datatype" : "int32", "value" : [1,1,1,1], "data" : { "section" : 2, "compression" : { "type" : "lz4" } } }, "pivot" : { "type" : "array", "shape" : [2], "datatype" : "real64", "value" : [91,147], "data" : { "section" : 3, "compression" : { "type" : "lz4" } } }, "longitude" : { "type" : "array", "shape" : [182,149], "datatype" : "real64", "data" : { "section" : 4, "compression" : { "type" : "lz4" } } }, "latitude" : { "type" : "array", "shape" : [182,149], "datatype" : "real64", "data" : { "section" : 5, "compression" : { "type" : "lz4" } } }, "flags" : { "type" : "array", "shape" : [182,149], "datatype" : "byte", "data" : { "section" : 6, "compression" : { "type" : "lz4" } } } } METADATA-END INDEX-BEGIN Ixxh64:343573c1da26565e Mxxh64:53a66ab9e53c90bah Pxxh64:f5d5c59a9a8881c1 xxh64:f37799e6d9c7d8b7[nxxh64:a1984bfda4108d5c}xxh64:2eaccd107c29ec9f INDEX-END DATA-BEGIN DATA-END DATA-BEGIN CP DATA-END DATA-BEGIN V@`b@ DATA-END DATA-BEGIN S@T@`1P@U@(!V@AW8!AX!AY0!AZP!@[@A\@!A ] !A^@!A_`!@?`@@A`?a0#1A?bp!11@A`?c!11A@?d`!11A ?e!11A ?f "@f e1?A`d(!11?c1`1@? b3?a3?C` 2?_@P`^!A]!A`\ !A[@!AZ!A@Y0!1A`X0!W`AVH!A`UH!AT!A`S !AR@!A Q#!Pp!AO3NMP1L(1Kp1JP1I1H1G01F1E 1D03CB1Ah@>81X3@AH1BH1C01D1E81F1G 1H01I1JP1K 1 L81 M1 N1 O1P(!AQ@!A`RX!1@  R:@;<=>?@@@#@AA =B"?1~ C"~!?A D#`# E 1]A{CF #^G@{#H1]I J@K@ L@@ M1 N1 O@1P(!AQ@!A`RX!A@S8!OT1`1A@U(!AV!W@`AX!AY0!AZP!@[@A\@!A ] !A^@!A_`!A@?`(#A`?a0#1A?bp!11@A`?c!11A@?d`!11A ?e!11A ?f "@f e1?A`d(!11?c1`1@? b3?a3?C` 2?_@`^A]!A`\ !A[@!AZ!A@Y0!1A`X0!W`AVH!A`UH!AT!A`S !AR@!A Q#!Pp!AO3NMP1L(1Kp1JP1I1H1G01F1E 1D03CB1Ah@>81>"E"K"`Q"V"@["`_#be#gi԰԰" Ӱ"`а" ̰"`ưA@I#"@"w"f"U#`B]"4" "` "@""VAO/x>!>2!711A>3g >τ=޿ a?0{?@=@ @ ` m1|("1`q~h1gA@_~^@ !X2`S~ ""P''"'#@)+"."2#7<"C1I ""@Q"`Y"b" k"t"~" AdE"" "`"˰" װ"@"""@"""߰"ְ"ΰ"ư"""" "@" "#1"" ""`""@"`G#@OX"b"m"y"`"" p"ర" "ϰY"@"@ "@0"?" O"]" l"y"`"`#"""`"Ű#@ɰ!h 1@DA N! "2@D!@DAJ!sAcH1OG19F! 1` ""Ȱ"॰"""`h"@""j#@!` 11`1Q11 W8 "Ayl1-1b2M}!<1`-}1@!1}1@1>H 2`~!11>H 1~2!1>1~1@2 !>P 1 ~1" "@"`/" B#@Uh"|Ae"`" ̰"` "` ""A ?e ư""""#u"l"c"@\"@V"Q"N"K"J"K"L" O"S"WA_"~!'1~1 '"@"*"B"[#`u""`ǰ*"`"8#To"""`"@հ"2!I2 &!6 "2@D1PQ !Z1a2g!11 1 1 1 2!r1PB`)FK# ΰCa!%I" 2FI"*"`5`3!1 G2!11`9tA`t833` 11{11u{1 a1Q{1E1@=1 }1@11=1}1 11=1@}1 ""`%"="@W1@r1>P 1 ~1ʾ1" ")"`J1k""૰"@4"""ٰ"`" # n"W"`B"." #` !11~1`>1X 1 پ1~1@>1'1 ߾1~1>1 1@~# ="]1~A\!~2 !#Iv"ࣰ*Ѱ"@.# \# A Vp!41@[1#°"߰"11$151A1J1 1@2@!1v1W101111D` 1` 2!E"2{1>'18< K"W#]!^A\1!11 11p?-!1qy1`C1 y2`"x!1a<2^|!_1 d1@l<1x|1@11<1@|111 '=1K}1 s11=1@}"$"T1>1~1"@"M"@"O"@""`°"@"l"C#!> "111~1@>"s"a# RF"="8"6"7";#BL#@Y!1|1X 1J}1 |1@}11%~"@d1@~A Z ,"r*"G""@ӰAW@!Z""@ڰ1 1N11@X 1` R.1M1g1z11 "1 &1111A |IH !41@1@~1`111AAO@ N"A 5 !1k1`A +!k1 PA k h/ܰ1 1@v1A1v11u1`1:1z111`:1`z111@';1K{1@t11 ;1|1A1`~1<"@2D!1`=""`f11>8"@D" s"+"@""a"`#1 辠1@~1 |>"K"A`}c`!=1H 1`2}!s=1`gH 1_1]}1^=1eH 1o2~}!=1P1{1 1|1M1|1`2`G}!""@e1~"@0*" h"ϰ2 6" \V11d1`1@1=1 z1@1݄11 26! 1 2 "b !1 1 1 1, 2!"1@1 11>1`_1`r1`2` ͰA7!f1!1@1@ 1@f11`5ݰ1*f1Dڰ?1?1r1 nAr^0`!Aq_!d1`81x1`11 81 x1ݸ1@1 ,91^y2!1:1ez11 1d;2{!#2!<1T}1`"&"1`~"`";"ذ" x"2n"2@=!1Q#@!<1 11`w|2_<!N1@D1@|1C<1L1`\1q|1 <1[1y11\z2 !3{11&|110}1@"IA`Yl"@A%X8 !DAЂVx !W1`ك1 V1̄1;111@X1@1冨2`!G1@f11`1111"1 1 1@d 2 ! 1@ 11 ADA!A#> !?A?:'A5013 118"j"۰1H'`!ΰ1PA`\D/p12eX!F1 ҽ2d}!<11@J2{!;1{1H1@{1:1@1ں1@z1:1 12 {!B;1`1Hw1`1Bx1@1jy2` !z2k!%|1 1@}"m*@6"ɰ"@1X11ڃ1@2 G!11.11@1`2 !{1 2!/ 11 11@_11 112!! 11} 1 1l 111` 1`" 2 ` !@11 O1װ2ΰ!Ű1`1g'1 V'A'2 !h11Qg1@1`Hf1121 r1@11@22r!1@R131s1 U1?5c@ u1V1171>x1 11@a:2 {!1`Ae=f0 &1`A-2^1@~"@KAd 1y|1;1r1 1@z1 ;:1@2!py1@F91`(12y!91+1I1ry191 1@Vt1 1uB@f]w1 1x12z!1`|1@` 1~I1 X 11!2@&!%11` 11 ̈1 AYS! 117 1 1` 1l1 1`1@2"^"!=1 a1S1`11161<11 1 b A > "M"` 1 -1`6A-1!6ʰ1111412 Z?`?'-182$b!)AI`^0!1^2 [!.2n!ɮ11.1o1=1@A/b/!Rp1Ѱ1 `111r1g10151u1ɶ1181`y1@12`<!}"h"r""1`=111U{1:1`1f1x2f8bষ1_w1 )711@1v161 141jw1`711p2@!xr1t1t2!v1 1Iy11 {"C*"_"@11 j11141`` 1@~1 2!oA`BR(!1 1@*AP !1`8$1`a$1K$1 #2[#1"J!d!1 2@e!1fA D0!u1 1@1y 1`=A`9 <!1@p` 1`11`1@13Ae'0"ʊ1h11tx 1݁?n_j6?031۰1yZ1 11` X1@1*V1t2t*!Dj1*1'A <*aPw!gj1@2x+b@l11S1.1n2!111s171i161@w1)1 p1;1`|#"O#@#1@12`y! 91 71u1v1`.61191t142k!P1Jt1`Y41{1@1t1@R51`x1`kl1{1n1@1 Fq11@7t2!gw1@1zA Y(>1 X 1 :W1 1@7 1ȋ1`J A๎T0!2 Y!111`[1121.O!.2 .!T.1-1,1+1Y)1F'2@$!)"1@#11:1_1@G11 11111ݰA/!1@D~12ɰ"G!p1C1@d@_@`+11^11q11^111`9 1a12!4$1`e11@A#*d@!k1૭12w1!ns1l1@o1r91 t{#@11`|2@;!c1Ʒ1Bv1411`^2 Oq!`02`!1\n1-A`a !1 m1-2!31n28/!ߨ1@Ja2!d12h!;1 m1`1r1 21w1* D"໰1@t1 &1 1@h11f2 !1@)1)12!: 1@A "Q!1J$1I1I1I1JI11HA FJp!gD1A1@>1:1@621",!&1` 1 s1M'11 P " 1 ~1۰1˰1vV+61`oAPH!{'A (12??L&1辰1<1|171@23!2%!W1 _1`=AHa!W112 D!5Z1S1 12_!h1J1J%1gg1 12 D.!p1!1181z1C` "#G9ed%!1s1)211 1@m1`,11@1i1c)A`!1`j1@e+1 qب1 RZ1iܨ1`^1*1c1@1@yi1|2o!1v1`T1@|` !W1W`7 !g1@11m141 1L1`#1୥1'A`5Qh!*A«Pp!,1qZ1 [1Z1SZ1 Y1W1rTA-QH!BM1@HACE!=1Y71l01(2!!i1A<! "2`!U԰11Y121 A@B"!|23 Q` !H 29H 6? '?i1 1?/1@1`)2p!$1`#1C1@O1>1 1`1@KO11m2R"pR"ē"M! 1X11`T2` "Nb!1@1 *1m11`161@y#`!1`1z1Q811@`1 q1/11U1i2d(!61;1se1$1}1@M2`Nd"$!1i1`f1&1Ш1MR1@Ԩ1W2ڨ!]191d1e1)l1@1sAY !{1@ 11` 11@֓1 AAU!1%1@X%1`Y1#+1 1@/AQ!31`,1 C61n1`n1@n1 n1@l2`1j!g1`%c1`m^1X1`R1K1 CA`;B12A"(!s=1@2(1"W11@J̰1 OAu8/0!kA&(!AGp1^1}U< @G1p1+"1&0 1*1.1|13@517 19 1 ;1Ƃ= 1}?H1@ķA@B01ӭC !_1hA2H!pC4xC6C8[!1@ 1 1`_2@!?1ɐ1`b1@F1 111 F11G1 R1 I101`1 1 ZQ1ۓ11u1`\11`1&1j11 61@41px1`C2 {Aye!61|11m1]+1@ 1 1e1@p#11ޠ1`_1D1ޘ1 1`^12Zߘ!1@`1!1@]ƨ1 'I1>̨1O1 <Ө1`W1"ۨ2 ]_!2Bh!2@q![1*{A ՄX" !i21VH"@!1$1ꨨ1,1c1`31ض191 15>12`JA!p1n11`2"6!|1w1@?r1 k2/d"["R!uH1=1 22`K! 21"@' "11ð1੦1 bA *![1 h1 bA`W 1[!x1u` ? @U/#@L' 1e+`1z/p!H1@1@8101 (1nϰ1`&˰A@?H!lް1ٰ1`Ӱ1`AͰ? ư9અA UzTH"1nU!&AOcV !gAmWW!aAGKX`!"A`>Y !иA 2Z(!A&[ !1Z1`2!6A^!1@c1^}11 v<2{!c1O1`;1|11 .1?11A`c!H2!1 _2V!љ11!1f1`U11 22Bw!{D03B`6e`x!q4@d^1$j1&'1f2!_12ܘ!1Y2![ؘ1"1/X1`1٘11Z1`;Am_x !>2{¨!F1`ʨ1`mO1@Ԩ1HY2`~ި!c1@V1n11 FzAๅX!m 111%1!1@1`+AT!)51|1=192`D!Ǩ2`J!2̨1M21@N!u1AK1dJ!AHX!191 Zx1en2gc!mW1J1<1@q\1=AL:h031#ܰ2 !A9."V" a" 1!1:A`^01+_` ?  1Q1=!ŰA@'1,10P!$ A`)41#68!% A$:1!<8"Ne1B @!B19171@N1 AE1FP!J0kA@Hx" k!kAK1_L1M 1uN01riO1.P!A"Q0!ARX"!Z  1_P1 y19AlUH":"_"!R1 1C1ڼ2`5!v1@G'1(1'1O2 !Q1H1y1b1 o1f1g11@$11Gp1@ů1`1/1`vp1@s2!41v1 )11%?11c1m 1N131 11a1 120!v""` A-e!v111 1`1 e1M"1`ߘ1$1Y1@D1 U՘12~R!Aa!Ȑ1P1FA`!1?T12 !31|1D<Ad]!F1̨2tR!ب1^1`\1k1@1Ey11`1m A@W!AVh"g'!1 {31@.1>1`è1@H1̨1PA@rQ!W1 ڨ1\1`亰AN0!ݼ11৹11 11 2`ؙ!nA`Dx!v1h1rY1`3I1 pAL<0!&A 7"Pذ!AK1p!ۼ1bk1A y! B yf `A@p1>1?XP1L14@ 1nm$(1(!1 riAet20!M|1(11Ձ1}1vA`6@8"0!4)#Pl!:1 1p1\A G!b۰1Ͱ2`!q2!ALN!x1421%'Q"g!1@<A Sh 1`1x11@j2)"[! 1:L2C!&<2!+141`1`1G 2<!m1 r1 1@e11@/Y1!A`O_! 1`$2`c!آ11 "1@}c1`1@&1 &(1 j1s11f41rx1 ؼ11`F1Ջ1@Q11\11`1.1@t1gE1@11kt1#/1`1W1@`11 ؘ2!R11`͘1@ҋ1cJ1`\ Aa!}1H1" 1 ɘ111@L11A1`%A +^!01Ҷ1`1=1 è1J2Ҩ!cYAZ("h!W1'x!؇21jWx!11@-1 ,1<ATh!.I13Ϩ1T2ڨ!^12f!1l1u۰1|ݰ1` ް1ݰ2ڰ!qְ1а1@^ɰ1`h1ٵ1`2@!r|B@j1W1ㆰ1\10A`F8!԰1iAu1!&1y+A%`!o1)*Az1Y9 P 11#s!V1@(pqAV)1}- !ΰA@=28!1WA` 81=:`!1@}۰1@gA@`A(!WAMC!MB151q(A)G"% !2p!۰A@YKx!1Ԩ1`,AಇO0!;1 ų1+2S!1~A`CS8 + 2`1!&h11`W1 AFW!2@3!F1@ 1 1B 1`11`p1y1^1L1iN11f@1j1251 Ͱ2!WU1`221ba!U1@R"`("1Y1 $]2n!51s(1#m1;1A=d@!1 11nW1@U1O1P,1Ks"˰11c1q1+11j1<[2 qc2J"!Ę11 BA1Aha!+" ``1A `!{1HD1`n1 Α1@X1n1 $A` ]"z2!B1XBA [ !AS1`ۨ2d1YX!v` 12 #"!$11N5AUh"E1T8!TAS !b111n1@#1 ow1 1}2!2Q!]1 1`tI 1BFǰ1`÷A`;C!1T~A,h@!kA`)p<!<A@8P!Ѱ1^1b1TAT)"v!i A@eAP1tH ?q@@[@`1!%@-*!aZh1@e?1nMA`DW58!\1i^A@[;1GU=h1J?X!eAA1*B!21`q1,cA fSF@!B1 11@1 1`1{1 Ұ11r11B1 111r1@ 1ٗ1 21@Vw11e1@19S21?W!H2*!11@1|1Is1@21L]11uH1 15" ("1%112!F1E2 !CAja!*E"x1~2@ !>N2!U1@52``y-2!31z11 1 Q112`)!q1F1 ;11`o1'101@Ț1T2 C!6ʘ11@A1)"9I1x1@7111`t2 5!11x2:!1 2!-11 1`&11`81¨1xLAZ !`11@uAtX!`8Ws)2@!h=1 Ǩ1`dP2e٨!b1+2q!1`2*"6!v1 ޑ1_1+A-N1!,M1")L 1#K1J!1Z2 k1UE0 !'װ1°2Ѭ!ޔ1<{1"1@ 1K1&A#5X!1@+N1 @11 A 01a@19 *@4?a@x1Vx1"1&+@Nh2 3@ְ1ܰ1 ݰ1 gڰ1OҰ1Ű1ڰ1а1@ð1@Ƶ2!11`oA`ZH!fE1/1l2 !A GM01;N!A JP"m!r61Ǭ1g#1 L1g !y1v2!*c1 ANV! 8W@!!2`!2F{!d2`a!2,H!101 Ф1(1;A@#^!~1#`U!A4`(!Fs1@1131`u1@L121=1`11 1 R1@z21z(dx"p!*11K1@R1 1'1mp" O162@Y!k1#Ad!Z1xM1A@VcP"$|!711@D1 m1+11N"@x-1 )Aw`!2m!y0"81 mA^!M}2@&!2@}"ڣ!.A@["D"Ш!C\11tA X!1 2`t/!1VF1gѨ2%\!2np1S`2 !j" '81 1 $12+!Y1["Ĉ-1 W1 Q1I1>2@1!!h 1o111ʰ1@11 1͡A@6]:"J!Ͱ1@o18A`-!$EA,$p 1 `!\%IA( (1C=}B?l'AZ 1mH"h'@X#1k.A E214W41*c61i8x!]jA`e<![1&12n!12߰"˰_F@˟!&2o!V2 <!6#1@r 2@!ְ11@R1[1"<1`R1&2`՜!  1N1u1@A``Ux!s1I1ؽ1A11$111-A`kY!ANZ!y1@11Ap\0!w141p1I1`\#!mM21!`!#`2h!s1I 1`a"@\1"PD1|p1 1@jrCc@`$!r1N2e!r1`1D11i16$1@n"^11ั1@g1 p1@x՘1 1`)E11`R1 jq17,1 11`11 *ݘ2S!.\11@ݘ1`1b1 $"@x-AX_!:1 jA`]!1 1떨14#1د2<!ɨ1 OW11`r1@1`3AనW`" 6!8è1@'P1 ܨ1 i1`1Q1! ANS!1`q">-11;1@1=DA*OEN M( !W1%1 |1p1 aB$OG0:1@"h 1X1ZA`AX!A~ ?!A8r:! "`1 w1!1Ak(0!K@A`21V!oj(3N`F?@6"A 1[9 !h26h1E-!"ð1ݰ1@11aX A;"! 1p1@%e2V!UFAn3D!r2!1հ11ݝ1 AcK"JF"(! 2!Ұ2 [!A@'BQ`! *R@u2 2'` 1`t1 1]1ADV !uA])Wx!1`B 1a{11@}Z1181>11`ȇ1I1wi"tJ"@`A`Z]p811%2`! 1@+K1`A1 81 1L11 +11$]1`u2!21Z{1`1 1Y#`83"c!S<2`!1 7!1 m"=1C1 1Wc1 AQd!1`;11 1[e1~2wژ"H"R!q1@Θ1 2M!1Ϙ""1@T2<1h_!hAA`^!;V11=n"@8`1`21[H!31 ¨1 Q1`-1p` 1@o1CW 27̨"Z!G1Bw11‘1ĩS`411@G1Ϩ1`OV1@ۨ2@`!ŰANH![Ȱ1İ2;1;J0!ߦ1"1 jFO2`1!h 1@71ư2@;!GA.:`!-1ΰ2@ql! 1`VM1A5#81h 1 1@fKݿS?ఔh1!U2ȸ-*@`A`f1"!)1[`?9@A=!1 12 $!1 K{#?!IA`-GX!l1@o1^а11$1 l1`K1 )1`< 1`{1be1A HQ!2 Z.!-1} 1` "s"![12@?!1@ 1T1@AmX@!1G1ִ1!1ގ1`;1 j1t1`G12@(!!1 1˃1@d1r1 A`<4`h"2`!51`x1ຼ1@2`G!11 1i1A1dp!XL11R131ځ11 1.l1E1511^1A&d!:{112!V2`W!?1˘11 MC#h!˾1}2="!11@RF1@ 11'1f1?2ͨ!$[`\@:y1 1`W1@)1 ں1K19ݨ1 n1i1"2@ !%E1 ֨1fA`U( !ņ12ۤ!21 31@L1ب1@Yb11[s11`~1F12 B!1@111xѰ1@1`A E1`D!;1h 1d1`qp=1R>1а1oaA0(1,!2 '?1Y1"X1:-xx!hA"1Z'(!1@kA`)28!=:X A@Q6!5a1:h1 gX 2^X A@7 1hAC"ɰ1YEX1F!pAOH!+1`1C1@j1R1Pp1J1%11`o11WO1lA U2R""H 11X1|A9V!`11/12 ]!1x311"(x11I11@"11@1ep2W!U1$1 ?B1M1 41Y12@p!{1`_1 11 01yx1@A c!V1@11=1@F1R1*2`z!Af`!jA1 h 1`گA@^@=p!˰1@R1ְA`W3!ְ1WA@'1="1Rj1Lj2m>? ?@$1A<#1dx(X!0@`ްU5@ &X !~,X 13X A@`1=@!^&X 1eX A{A`!i1 1}1Aa!VD1.1@_1 1_11K1 A1@29!-19~1K1 1r1"2 v!i12 <Ԩ"j1镨1+1 1W1@g1`i11 !1@=1`uа1tb1@1 1M1ǟ1+"A1@>1Ũ1JAOP !2`癰!(11t]I Bh 1|!1 1ѰAC(!nA>7A1H?!y11k1 ް1M1఻1`-R1a.1`oA`|1p`B? !\ĸ1@BA@sa%P1*X !}A@1!2Lݰ!AA@ :!WX A >01I@!11 а1@A D(!#|A+YF!J31 11A`jJ!*VA -&L8!1Ű11g1 1@11`w^1`1:11 1A@BqT!1Q1`c1z-1@AW@!k1@F191@11i2!3111@f11@U81@]@}12@_\1,K11> 1`_1f11`i&1k1z2@!D11 1*1z2!E1@n"$'1 1j1 411hBf1`T2H!1 1`V"`"2 Ǻ!n1$Ab!1`M10 1Ř11 AC1`1Ř1"@ 32"v!0=1̠1@E]A`V]!1`\A2aؠ!so11 ɞ161'Ϩ1g11s1`01 ɨ16a1 1"`8A(U ! T1@01o}"`11~31¨1P2ܨ!f11 tA`O"!`1 1=۰1`fȰ2寰!1m1@Dh 11FA@}BP!gA$@!꺰A`]%;!ӉA5!E2 1B+!8Aa!H!~D'A H!Eh"?@D !p-Am5"!1 1)X 1(lX A4(16`!11X A@ >!v1@i1GW1#@A$D(1E!1ù1`UA bH !O2AxJ8!1 2@vb!d,1 a11 m1,181 1f11?1*1 1 1Op11 wN1 ^2`z&!1C12]11 G%12`!L1A`Z !r11`@:1ୟ1`v1n12D!"`ph 1O1 2 !8"@h 1 g111`iK2Ԓ!1@&1@`s" 1`M1`b111]11@P 2oa!1y1g1@1 -1`21?1 1 +1F1u2@ϧ!Z281Ybp!}2|6v`1 1l2+!1Ʈ1Lr1`711@h# 3!a1O;1 Ϡ1Be"h11 ,1LŠ1@ _2=!̓1.1ɠ1d1`41Y61_Ѩ1`3l111:U`]l1 11W01@Ĩ2W!1x1O1@2 f!t1&1 Q1 O1G1@$:1@ &1 11ð1"A`Ep!(&h ABx!S1SA8>!l^1`1`)1`?ȃ0L2`1 h`wx1` "!p-AS12`"13 1 1[i30150!HX A`99 1<!F A@"1iA!bӰ1`114v1`1͢1`1 @p11D11`  1 "Vo"!J111`1@81M2!2n!1v+11 AjHZh!1 1@l2!?61ڝ2 !s2 !#S1@ 1$>1 Y11 z[1ϝ#h !(1)p1@(1`2S!1@A1H1_1@2AK!1`s#8!/A f~B D>f1ϋ141`yݠ1 O1@411j1CAPb!1@c11 4ט11Q1 1`Ҙ1ౕ1 Y1`2 )̨![2"&!v1`M1 E1`ޠ1Ny11`1@GMAgZ!1%1è1aA /X`!N<2`Lڠ!x1`11\O2`ex!11R11~"`u1.131@1;L1@Ԩ1@5Z1`͸Na1/1AqJ 1MIP!"A@4F!UA`HuD!O-h 1ް1`1`_1 1۰A 6p!;1$g1"A@y%1j!h}"A@f81X 1x"oI1 !21j/".X !~X 1 X 1X A;P!.$X 1`a(AM@0!1q1Y"1@12 Lİ!1^2`&!2!+o2.!114f1#1A~QP!11uz1+1 J2!#" 11Vn11G11e1A a\(["Y8"1 1|1o1k1 <"`g8"1v1 120J! 12a1!˂"K81+1@!1`+161@"`pf1L1 A K f!hg1`:1 ݠ11,'1͠1(v1`A 1C̠1Gz1[*"`OAՐb!:G11)1v14" 81@ٶ1@Jz1 @?1@A_!{,11@V1 1 1@&#1r2 \10[!P1:1_ڠ1 {1 1A1^1`g1 B1@"xw1%1@Ũ1e2!&1`@1ܨ1w1p1]1u?@R@XeA`Q!2k !M11`\,2@%1XMH!A1`11֍1Vh 11ΰ1~1@]'h 1ȰA>`"x9("@@1@YJAϰ*1$8 P!Z}@S?7`1A"@G1`A0 1W3 1{5 !X 2 '!zJX A\>(!/A(A! 1K1@1`LX 12=u!C1 y 1D1@n#`hX !11@܂19112\! 1f11 <$1Å1`1 O1 1`_$ 1@ALmT0!A1B1y1 1m1A`'W8!1 1211b1@81@1 CABZH!11W2!1 .~1F11 ߠ1:1@`%1`ɠ12m1ANY8![1ᤨ1`I1\1Β161ڠ1@}1: 1@1yb1`1@1<1@Uר1'p11 1,2z!2G1 Ϩ1T1I11`11Kah 1J81@1ɰAFx!4h AC8!z1h A6>?0cM<P9!K1m>` 1-1`_71 A0! h|? >2ೀ"!31X AK/1+2!X AD612E9!{X 11X 1fX 1ذ1fϰ1Ͼ1@1l1a2s5!2!1tP1X P 1@1``z1y-11]1<=11Ě2_%!A~1l1P41[1@1V1@)1 & 1@ȓ!7l11 :>1+1`1_2¹!1g1AX! d1`1 1@(_" -1@ "1>1 1@1`Gq1 ?A@1\0[1i2 h"B!EH1Z181@2"3b"@!A?ah"!11E61!11`E1`71@1 j1141`1 01h1`011@6̨1`h1@11{I1l11 8@1611@sO1 1d"X 1 c31a2!v#!2"*!1E[A]( !14A(\8!y11 @ 1 Kh1@1m1?^1@h1Ϯ1 ^WAࡨX`!0Q111`I12ഗ!=111 )1?˨1k11 2;>!/ը1Wi1 s1@1~2 蘰"<!s2"Ah 1lA@K !3°1$1%Kh 1`1 MFDڰ1@c1 ɰ1 庰1 N1}v1`wF1A)1L#P1W1Ɇ ˿ ) !3G"2u31'@1,"uX !]X 1`r1g1 X 1`eX 1A@P!z1 KA uoC !{S1/1,1 A`{G!_1@q111=111A1A`N17OP "!VC1 -2`Q"=F!u1~1`]11J) !1@k1191@21OV!1h1CK1֛1@1:"1h 11)1@z11 1u11@%1`1A5A\!A ]H!Zy1#1I]1`S_@1 /1 v"2g!^11 1d1@gA!c"!1@~R2!'11"-1 a*12zX!x1`1.1Р1)u1@d1Ș1@Yw1`)1@ݘ1J1O2` "L̐!91\R2`V_@U1`11"1`"q1P1K2`Q!12kL!1@^1&N1 O1`1 SS!1DY12 ٱ!]21V!v]2e"5"T!119>ARS@!y11ꪰ1>1@Ψ1F[1 1 gg1Ͱ1 Bh AԪM 1:L ![1"1ܰ1oh A@\+FX1Dp "Gh !jİA6@ !,<("1Z11TA@/p! ,A Pp"1?1 Z?P!31`h-A)!$dX 1@\jX 1A 7PS9@HAN>!@R1@HX1@S2`lE!-AL Ex!2!}1@2`!A zfJH!1 2f!C 1?1`L"X 1 *2i!)d1 A@Q!?Z2@K![ 1f1@A ,S 1!1iTX"!>31@"2hf1`=1o11+1x1@Z"`[1 aY111=1 ߋ1@}A,Z8!'" 1 -11 2G!Ȭ1`2@`1k1Ai_P!1 ?1a2@]")"B!N15112``!-181@/111"M>1!2`![E1ڠ1r1c1`O2@O"!Ο11q2`£!U111@}`1 1@p1Z1`1`џ1=1ר1m1`2!$1ϗ111`!1 Ű1h A@@9J !{ڰ1`jh 1+1`Xh 1@1@h 1x1YA`E7!扰1`؉1(3Aq h1H uA@U!;:X 2ϸX !VX 2pX 15 "!d`==@A  Ax !AC!R1$1aը1`թ1Au1 a811q1V1'1x1,A11r11``1'"`'2@!!(h11@n1?1@e1!1&10z1@r12 1ޘ2`!g!nA@$U@"py!1 l1 [1`t131`&1i2"!/1@ s1p1@$1fD1 z12`#&!Fv1V2 !y1M1c91+1 1}2!s1?11 #'!1`2 d!2 @-!b1L 11 1pu1n1t1`1v1 1 M1A`՝d0!Z,11W1 11 =2s"!M1`21@511GD2 !1}92@WӘ"Eq!1@㷠"X1 1@ȷ1g1a2@X˘!151x1V1]1e1@PИ11`@E` 11@[u1/1`12@[w1mʠ1`1142!17H1 a11H111,1ư1[1 X1pw2!{161@qh 1`ə13Z1@AwIP!,1 9h 1 @NC1b1("A@ 7@!A `/01wO'!IA!\ 1['e @ h0!iX A+-1$1!X 1`1x1"1`ʘ1X AB!1@1 11ਿ181dQ1 1A`RmJ 1sKX!1 N1 A2wM( !12`!1 1 U112@!_2!1o71161@6 1xATp!e1@2!.k11`b1<1}2»1`610t2ற!2@.t-1 1 2"7!}1@A`Zi!c1 1b1@1c#1@2q"m"!k1A`r_ !1g2!1O1 1_1 1Q1`A@Wd8!z2`hi! 1@~1 1t1Y"'111@#2!]1"31^1"٨11@J1'1 1P2 M1@1>1`ט1u1`R1 C"191ɘ#" 4!1`o1c1!1@P"`1c1&11 '1ct1:1@Š1`1P1r1`@٠1#1]12ޠ! 1X1>1@˨1u161`" 1@11^1U(A`R 2!e11X151J1%h 1 u1*1eh A@F`"DѰ!ܰ1ͰAIM=p!ְ?4 _,A`%1˘vTAS1^\&1 #."M!-1`f1i2t8c1#1G NXD@gW1 F1''2`1H(!~1g11 F1}1 2H-1@¨1F128C"q!1S111V161`o1b1@`1[!1`:a161@1@ 1zAaT!1p1 H1 1(11 U-1a1•1^1@ 1 11e"H"1A X!B1~1I12`<!111e"(1`1q1 1B11 ]011 C11`Xp A1`!}11 VX1{1JU2`"s!c 1`s1 N" 1j1@˨1J1#!" 8X B?c!bP!61=Ԙ1@%y"PA`!܍1@=J2* 1^_H !11ɘ1@g1 11`^21Q\`!x1`11z1;1`j1d,1q1`1y1 @18 1Ҙ1 1&h131`̘1 ٗ1c1d-11111 L11`Ӡ11T1@1Ψ1@2="!1@?M1/1_1D61>θ1`_11k1H1R"1@k1 A@%K !|h AG!h 1`1@DAu>!("1̨1A(A$ 1j1$ڸ1Tx1y!"UX (l1@N=5H!ɨ1A q?@!OP 1 LD1{xA@DP1EE!1`{2R!y1`A Jp!414AkL !=1@7A@h N0!n1 14z11`X.1 d1W1`1" 1@=j111`]1?1@y11`AlGS 21@^1x1@1`21@j"h AU!Q1`01_1L11`I11L1f~"[02!1T1@s1J1` 1M1 1$"`""1A7[!1 11@11!1o1 Hx2!1`1{AGb(!_171@Ad@!\?11`YAe("!d1Ϡ1`E2 ƨ!P11`>1'1Ԑ1 S1@A2!1`1@ޯ2M!1 1`aH1Q1`1h2$!g1L1@Mg" 31`2`!1O1\191@d1`Ɖ"3(32d-!2 Ҙ!ͤ19v1F#h !1 %2D{"D!- "̸11 [2!$ۨ1 1P111ae2`t"A!!P1X1 w1@\1-|AYPP"V!`p 1` 1 Al1h 1h 1`Zh A`MDx1mB!Ay:h!3HD1x:A`#!xh۲? H!jX A/h1!4!1@P"kH-1oH1!u1鼨2!2!1@Ĩ2.!N"@81 1@C12d!#1ji1`1W1 1`11@N11@I1 11 B1`q1d1D11@/1@a1R1!A= S0 !O1P 9`f1[111+1L1<{2!11&1 P">111 Ag(Wp"\V!1111@U1ҏ11` 1)Q1o1p1@811'1`V1`A1s?1"0"$(1`£1e1`1 11`2RE!11g1`-1"`O2!tʨ1 11@1 lA`aX!_1m&1͐2`|!G31@ߘ2Cd!P2!&1`˘1v1@%1`ِ1@1M1 1pΘ11Y1!111`1T1 w$1n"I1`"`w2@M!&` !w٘1e1`U1`1 61 1ۘ1"1`aH1 E"(1즨1`Zm1 1AEU!/1m1 &2Kڨ!1f41ٰ1w2` !1 1@q1`=1d1If1޸2-1OJ!@A E1C1.@1;!1 k1Ex-A HP@ 1֪+!A B7!Z1LP 14A@{B!1`lVA@jxF"1}sH!?R11 1yA3LP! 1]1`1lf1@Z1OT11 m1GI2Ez!1 1Vx1`01[1@R1?1 12`3-!QX1 Ä1`?"ո1 81]Y 1@` 1aW1̚1nAU!n,1+U1-|11P19@V@ $6#@P""!e1 1@1'#1N1<|12#!+1G11@o2!F111:1@1H"w81@1cwA]!1[w1L"cp Ao`19aH!"h "O2 9!e1 @V#f@;L2 !;ɨ1,!2`=gP"4! 1H1yj1 I1@1~01 1FS"H 2ۗ!3DA"\X!1h2'!I1{1Gv1`@1 1@ڈ1@,1c{1@N1O"111ຨ1ڃ1@`2?!j!1`~1&1@%|1 EW1@01 AW !1 2U|xD1`y1 1`1@Q2!ب#`x !S1 1̻1 $h"x 1 1@B1ϰ2P!mŰ1@ +1p1`1-Ӹ2b 1 I1M{Gx!h 1`A%= 1N6!Aɰ1@Ux-1p-A`o!dX 1`=X131!1@1 g`HC@`ŒANEP!8A`#$HX!1o13Π1`ދA`:L!22Gr!21`!11p1@1 "1iT1`Ã1 *1@AQ0!/1W1d~1j112@!81`]1҃1`Ԫ1 11 .13e 1 1Q1 ˌ1`Ż1@1` 1`-1N1 o1@111`A@V!11 jS2v!ƙ1@11 1J61a111O2)!c1C1N1-1d|1#20!A * [!2!=1%o1 >1`H*"^A`2`!g1`/` 1 S` 11$Asf!171 '1^2!1?2@u!P1O1`1`{l2@&!v1' 11@T11 2`r"&!@1b2`.O!P111 /1Uh2>!1>x1>" 2!/h1K1`+11@/111@V1З1`y#!411(11@1l1y@1 ~11Ѱ1@|1E1@ 2{Ψ!&1EGA,T(! "`P2!T1111`_P +1@׸12@I! AcCp !@A`m78h 1301M  5` !I'A?4(!lF1 1`o1 1@!1 H 1`Ψ1@11 Р1̟1[171|1@b51 z181hA@ P!?1Np1`"11 IAQ !WA2`9f!11 e11p1@B01@P1?p1{1`?2!1X1:D14s !-A JT!|1@1R11 [11;1V1q1*11=1p"@B@2@!R91X1y11@\1"51@51b1 8112 %4s-2 i!1W2"!1܌1`1ޢ1 L1n1TA^"1(Ը2!11`+2v1Ae` !1!1`٠2!"!1 2*"!M1琘2@M!15V1%11@Xl2*!1@"`E0-1 L1 H1x1`1@1s1 L1/'11+p1#8!ր1``e1L1 61 #12@x!1T1@X11)1@Da1`\A111`1{1`1d1];1"2!1TK"f1cհ11@rLT`ʩ2K!1 o2!V112`!=1@eyA.H`1y;Ep !]h A :10le]ZA%(!A^9!11 WH 1"ߨAfG!CA`3I8"F!̠11 Q1 )1-111Bp1`c<1n11"`ʘ#@6!@p1Nd1`i1@%1 2(1h1`;1 xW1`r11 &1w1@111A1@91\1g 1$1A1^j1h1m1#1`k11 1`1@s32I!s`2Xw!11@1A2@h! 1)1G1f1@1a1 Y11!"X2!@1@#3!v1#H(Y@@@(1w1@511`A`x]!Ǹ"`AjaH"! V1 *1\01@i3(A3b8!yA`p!`1 M1,:181`611 D11P1Q2"(Ӏ!1 g171 1 qp2D!1um1UJ1t(1@2`!1$1}1`us1Z1D1@821"1`2 p1{p1*1@711@51U11q1`T#!"1T1 1@1 2m!G12`"!" 1~d2 ,!h1\1`e1$1.1U11`^2@Ÿ!1@>A@@PN0AyG1$\C^< 1`1B!Q>h-1|'1@2@- 1pB!ר1`H 2Vz!ߠ1@1`g11˜AtN@"!'"M,A`JPh!;1`p1 A5 Q h1 U1fx1C1@l"p81kA RX!'1A1`Y2Xq!2!#111h16p1'1 ?"1 u1@ ! !61pV"t`2j!1`#)-(11"V1@81Z*1<1O2+c911 1#1 111@1<"(1}1 2!1$1`#X" 1AUX!n1A V;Y!ּ2W!Z1@22 !~1h(h 1@S1˸1`2`[!G1`u1@WL1`d1F2!!_1໥2 !2"?`811@x2Š!'k14:1@4 1h1ż1n2`.v"U!611H11`"11y"2J"6"&!"P 1 2!`"G1@q2`}"XAx 2 u!1@2`mN1 :"@! 1 v12:"H!щ1g1`:C1@q11`2 ؔ!:_1$A`=UH!d2`D! 2@Hv!|1 DZ1`911r1m J@( A R@1S21ͱqO#@b51>1CC!H 11Kݠ1Q1@R1S1e1@#1T2@v-x!n1ܧ1`10 X1P61]1o1`Q11o"@5O1j@">2 G!f^1t"`Ɛ1`Ԝ1C1@,11"`1d X2@ט1`@1BS1@`f1 z1@1` 1 V*T@Fh 1S"H32@r"ˀ!i1๛1@1 11@]1`1 1U@2O&!^61#G1 X13k1~1q1 11 o11}131 wW1~"` 1`11}R1 1`i1 W11`VhA%Z!1 4^x 1 1C11A01@%Wh1"1Mt1@A2af1_h !ə1 ݐ1 G1b͈1g11 x1" h11``1s2!1`(b1D1~)1 1 01@1`12 ɝ!݈"321_"@_27!Z%11 1@"`r1@=11 {H1"1T1 111 v"1 L# x3!"w16b1L171n 11`1`1@1ם2q~!\1`71A1V021w172 -!B121T1W"26f!t1@<311@.1wh A F5 x KXA=!̨1C1`̠2!S/1\1Q1x1Bd11@1n$`"} 1;11@1`1 4 01%1 >1V1 l1`D1@̔1I1ݸ2!1@ 1`sS@{10"1/1<1J1W1 Fd2@q!41Q"@881`a" 1} 1@1`m+1 l71B1K1U1^1f"`i 1 x"1 e1@?"E111շ1 1@2 "?s1@XA U@ 14(181K2^!Nt11 1O1 @Z V@ 4!e2`ٟ!1X:11+11@1 B1Dyx 1 1`11Fɨ"Ƙ1Ӡ1oJ{]`ʈ1D2 #x!1,A12X!1@+" #1` 2@b!m111`21 1@Ѝ1 y~2o8 1@R19C131$111" W1`"`s#C1@0"@#="1F(1`1@m1#@x3!Ƽ12=!~11/r"S1Q1t@1`.11"Pa"`P"@"K2{"V!*2@.!Ӿ1Ly#ܘ !1512`41,R!rx 11@B@ @h'1w H 1z1`P8 1AAO![1@1`1&`2l!11 1@<"`߈# "!411@1"`e 1@12"!X(1g31=1*H1Q2`|[1m1v" T1"0 1<1 11@51`11 !"%"z!b1 1@l%1)1`g.12"P1 ;;1 ~?1`C1(H1 L#+"1`Z2 _!d1dj1`p1#v"1A"tp")"t(#@j8!'("" "~ 1 "J2" !/1(M1Ip1U1p V@``"111d5AK\(!h 1 A`,Z`@ !2@!x1`\B1#P1ߟ2Ij!?1 |5YL" (I11M1@e12`_1`n1(e"`22AS!J1B1;" E2@+ 1*12x "@@"` " #!tH" UH11n1}"11`""`232`ך"x1X{1`o"1@R1`B15/1 1AW8"}!2` `!2!>91zn11 Kx 1O '1EX 1hH 168 1`E21 sh11d1`.(1 d11 ײ2!1`,#P!!"@ "{ "`2H1@R #!e" H" 1 |11݉11@2!H1M1$1ծ1c1 ׷142!11H1111`e2@! ypzmP  0@P`p 0@P`p 0@P`p   0 @ P ` p            0 @ P ` p            0 @ P @          0 @ P ` p            0 @ P ` p          0@P`ApU 0@P`p!L0@P`pPT@ DATA-END DATA-BEGIN 2S"q1U1r91OeR1ݰ1@$1Y1`gx1DT".1CASQI1 1@`1H`1`21O@ P"٠1 m"71@EAO>1O`N1@"OǡM1O&L1@D1>vKIAJ031GAI` 1 OaH1G1 ARF8ATEA`D[1>1VC[A@BIO 9A1@1@At>03A`^<-A:'1&9AS7`fAz5` A@38A21A|/A ",>A@(A%>A"pA pwA cpA1A {` A@2'AŒ 8A`Z\`fO|_ tOOS?` S?S t? "S|@-O`Z\OŒ O@2O {O1A c` O A"x-A%A@('A ",pA|/8A21pA@3A{z5` ("|X @`"}H @`#~8`#8@`#H @ `5 ` h  (@ (p M x @` x  P5H `@1dR7p"e"g" j#mq"@v"{""## ""`"# "@""@1 S""!"-"@9"E"P"\# gr"|" """"`"" "@#"`"@"" "@""""`  0@@"0`p"@"`" """" ""@""@0" " "`"#`~"z"@w" t"q" o"@m#kj*i0@P`p 0@P`p 0@P`pP 0@P`p `9@""""#! @"#"@A"a"`"""1-!@"@]""@" 1`-"("d""1# "@H"""1+$(" c#`1% "3" d"`""`1&0" 5"W"w"" ""`" #"'P""""""`"P"`"""`""#v^"F#,#@##@w" _"G#1" ""`#"@"""#` 0@P`p 0@P`p 0@P`pmO`=I 0@P`p3>zAQ:"1`t1 "1>"1v1@"2@H1@w2 2'11"`1@'"@1"i"`1"@m""1, " ]""## """"#`~Y"."#"X"@"""`N""@"x"1"@(#`""X""i"41 1"""`k"@R"`>"@0"'"$ 0@P`p 0@P`p 0@P`p1@U}H?zA< !11Q11ҡ11\111'1`e11ѫ11@!1;1`L1Q1K181@"1 1 a1"1`""1K"# "`#@""" {"9"``p"0"Đ" O"Ұ"@O""9""@#"T" ("."@""@X""p"@.1ǭ"h"`"ʠ"`"U"`,""# 0@P`p 0@P`p 0@P`p1U}EA@tA@ "|1C 2E!G1I1`#L1@aN2`P!R1 +U1lW2@Y![1 ^1/`1Db1Kd1Bf2'h!i1k1Ym1n1 [p1q1r1t1"u1 v"1`w1!x""10y"@R"X"D""0"o"X"o"p" "_" 1sh"Ÿ"@#!o " "1`l"1 j"0"`@2gX1`e" "@Bx1c""J1 a#@T#`wW"L#`w0@P`p 0@P`p 0@P`p"@j`|! 1@1V11@h12!E1111 T2!1t1`1 1 + 1` 12S"!112!1 I"1`{"1@P"""#`""w"*"P1 \P"p"@N"`#` W""` " ;" g""1 h"@">0"r@1 " X"9"p1  "Y"`Ґ"X1 "`"F"@ """  0@P`p 0@P`p 0@P`p1u%1?1`LX~}#AA` z12!1 22ٻ!"@<111X1`11`@11G11 1H1q11@u1 M2@"!1w12`!119"1/#v"""`N"8"" X"P#@p"Ӑ"ؠ"Ͱ1`x""_1(1 #@!k"+"(1@"@1^"BX23!3"Dx1@f"1@#F!" L"`"""  0@P`p 0@P`p 0@P`p"`P1@13X`{1`l2Dp!s1`w1Q{11`1@11S2!˕1@r11@11/1X1`1`D1111B1`U191h111 114"`""" "@X"8"9" qP"`"wp"J1`"1`&1@2 O2!ٴ"1 P1"`(1"f@1Ũ#3X!1J11"11""[1ޜ""@M" ; 0@P`p 0@P`p 0@P`pX#AB` z1 X1 $1 )1`-1 31'82 I=!nB1G1 L1Q2V![1 r`1#e1i1n2@Xr!iv1Iz1}1@m1 1@1t11E2M!1 1ޕ11@1$"f" l"5" (""@?@"/P#h!H1"`/1P1S1:" "1w1"`1T{1x"`(1At1q1`o2m!k"p1`g1Ff1d2gc!5b2.aU1_" 11`^" 0@P`p 0@P`p 0@lH`p"a1 NX`{1 hɘ11112!o1 !1ArCh"! 111@14$2*!/1R51:2?!ZD1 H2M!P1~T1 W1Z1 S]1_1a11c1|d1se1@f"k" n"#"`(#8H1`)b81`1 ^1\"S"1`JU1R1O1L1 I1`F1 C1m@2@P=:(1 171941Y11 .1`+1@o)1'"`2 "!,!1 "E1@)1J1"J") 0@P`p 0@P`p 0@P`p!("'1 ȀX}Ah}1 k1s1{1 Q1@11`12j!̵12F!O1`,11E1@t1]1@AHD)!@1 1 11}11@D"1 %1e(1`*1 ,1`w.1/1r0"`" "q"`("@81@!-01[+2G)!&1I$1j!"R2`!1`191@e 11" 1 1@1`11 w1@1 1@k1 r"1`.1"2;!11@#qH 0@P`p 0@P`p 0@P`p1"/CC=P{"`1t1 w""1V71A1 6L1`V1`1 )k2:u"!ƈ1 01@Q1"11 1q111@2!1@11`1w1n12@"D!4"" #*("8# HX1@0@1B1 1s2@2D!1/"`r1`11 ػ11`1W1 "(P1̟"h1 1 2@!|1 "711Z1 1`"f" 3 0@P`p 0@P`p 0@P`pX|}~!v2[!1`պ2 P!#x1'E@y11*1%61xA1rL1 W1`9a1j19t1|1@:112!1 1`1ܮ1c1M11J1b"@""+1@ "H8"H"dX1`B 11@1`p2̞!ݙ"`1B1੉2 !~2)x!8r1@Jl1if1@`1Z1yU1`/P1!K1@YF" ހ1=19"`1320!.1 &-1+#@9 0@P`p 0@P`p 0@P`p "h2}2 !/""FP1 [" H1y1 1" (1@U11`112bF@c11'1@21;1D1:M1@T1[19b1@g1l1p1't2`v!x1z""k1`y".("81 ys" BX"h1`Ch1c1i^1X1S1L1`_F1 ?181@11*2# 1`111`1 0`1'1q1@1"1`2!1 1`R2R!"9 0@P`p 0@P`p 0@P`p !Q"qx}}1 p",0 1 *1e1`2O! 1`]1.2 @(2a!4r161Ƒ1@ؠ1^1O12G"<!w1A`GP" !1@1N!1`&1 +1@/22!41@!6""`}15131`R11..1@c*1%1 1o1 f1`11 101_1 Y1,111L2`!1@ 1U"``1 љ12̌!1 1|"" 2q!o1@m"" 0@P`p 0@P`p 0@P`p !`1 x}}1!1@51`H1\"@ 1 "`A1s11a1`29!01t!"P1D2 1@e#aH!:2@!"`# 0@P`p 0@P`p 0@P`p 1c "x1|"`u~ ~ }"ۈ2$!=2`W!cq" { 1 11>1`1 1@Q$1X<1S1`tj1d1@1`1 1K111A` I0!e1%1>01@T91 .A1G1@+M1PQ1>T1@U1V""0 "_0"@@1`H(1@B1<1`|42,!"151111"1x1 11 #@H!u1 1`~1s1@h2^!2U1XL1>D2<!61@11`q,1(1`E&""1 0@P`p 0@P`p 0@P`p>` 1 `"]x1*&2@2 w}1wh1!1Ͱ12!1`>%1:B"1@Q{" : 1`1R1`W1@01"Z`1F2`'\!ip11@e2 !e1q1(1@111\2@8!AJ!1@$" Y" N "` 018"@P"}`11 Q1`11`1112@!,2`r"jd`HZ(1`m:" 1?1*11wx11@B2H!'112@w!O1`B2 X!1 0@P`p 0@P`p 0@P`p 0@P`p 0@P`}}"w111@1r`"@ 1#1`;1"`]#18!e=01Z1 w1[1.1@1 1A``J@ !W#1617I1 Z1li1Sw1 11 1؟1 -11Q1+"`1@1@(1 m1h1(1`#/p!1 z1n1@Jb1EU1G1\91`*110 2!1 Z1@1111 ג1Z1x1l1`fa11W1N1E1>1)9" 8 1\11d/1`. 0@P`p 0@P`p 0@P`p 0@P`p 0@P`} 1 Q"!""!"`m 10"`"`^1 &"X" @1P1`i1@t"1"@1J`1 "1E1g1 8"H 1@g1" ` "`1 _111" G "@ " 1 \@1\"@sp1`"[ 1@52"s! x K@@1)141 >1]G1 %N1DS1V1@X"`1W1T1@P1J1 C1;121i'" 1@1@U1`1`1 1@111ː2{.2]! L1r;2H+"11`:112 T!S"@1`,1 ""`1 Q1 !"@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p1`81 9"x1@" &1)1@j:1M1d1@|"W" 0!1"P1h1@71["`0 1E11@31p"@!1h1#[b1(1%" 1@l"Ҁ "Ը 11 51j 14'1@A1AZ2`eq!111 1w1 $11@1<11@A1 "9"1`n11 C1o1 51112׫!Ӝ11B|1 j1`Y1F2 3!!1 1`01"n1)1a1 511 "1tq2c!UW"1%B" 121@,1(1&" 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p":+"`n+1;`" &"0 1|`1`Ms11@"p!11i"1H 1?x1d"+8 "X1@!1P" 20X0 "@ 1D1X" 1=T1|"01`1l1`@p191@:\2`0}112@1 Lh"!!61FJ1[1@=k2x![111@91`1"" " ("81(1ׅ1{1`o1b2`T!TE141 r#1`0"4 1`"11 f1`1΃1'o" " "Z1 "` 1v1c1s1 2m!"*1e#I !2@<"Y 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p!cP1a" "1 2`! "@A"`+11Tx1@Y="r&1b#(ՠI@(" 1`1@1R1%1C" 6 1&11'"(#8!s81@1`"= 1 1f+1K1i1 r1`2ٹ!2 J!ArM!1@#1`A-1A51 #;1>1@"@R"`19131+1@7""h1`. 1 2W!1@11 #H`1b1L" 1l11 ^H151 |1U"1`K1v1,e"@1E2`8!,1n"1 1I"`{# !  0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p2l`-'"@!12 1@#`ޠ0'!"+13"`,1X}2W1q1@)"@ 1`0"~1P1@tx1@K1 |1`"ౘ #ݸb 1m1@1"?1h1`oF2m!12!b1@"Jh1O1g1}1`1@61v1k1@1m11Q2P!111`?11 T1U"Lj1`Ȁ1 tn1Z1@HF1 01E1 )1~1h1"11q1Y1@UB"`p2`O!11#`S!&112@!]11@z1 w1v 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 1p11 s27#`0X71#@X*!oE1`i#2Ԁ%111D1u1@$" Yx1>"@ 1Bx11@"1I"[11`1 "A@2p!".h 11G1 A1f1@11'1"1`+N@ )1`;2@K!^Y1 d15m1qs1Aw1x"@1t1 o1 g2]!JR1@D151@$1%1`D1`"0"o"8`1E 1@p1V1`<"! 1 1@"1t"ʈ 11`s1W]1@H151@h$1111@11@Q11  0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 1Ր"@A"O6",2@(!wx"0"$71cv1"P1"`-!1@%"%1 1ڿ"@8,"`@1@d1@1#Ix!!LD1@{1 21"" ]1 "@"0 " 1Ip1!v1@1f2` 161V"@oH11`E2@c!111 AOP"! 1 1`"" #0!@1`111 z11 11"@+1\1B2`(! 1111/11e1J1`.1" 1`1 11`" 1|"@ 1a1bV2`M!G17D" 0@P`p 0@P`p 0@P`p#@p 0@P`p 0@P`p 0@P`p1" @2* " !;1Z" 14"\1"+1@-c1"@%181A"`:1?" h&16.1i2! 2 v*"1#1d@1Z11"@ 1`O"` 1d1 2b!X1L 1J:1@Q1 f1w1`1@ђ1`1 1@1611@,2`P!1["@x`1ug2tU!A1 +1`"10"MH1 J1P2`""q!CS"@ 11`8111@11Rh1`N"'P1"Ш 2!111`M1D11Ҥ1` 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 1@X"X1@1@*1A"@a881`h142?!1 31g1131`1*O1["U(2"G!11@"GH 2@!11V31@l1 -12@p`@1u1?1 "` 1/&2YL!o1`א1@2 V!1LA`kP(!0 1t181|2`F2!101 1 1@1p1@W1 111 =1we1H1*1` 1j11 [12k1[,1@t 2[pAx2 U!ƛ1`Â1@zk1 V1B2`1P2 !1@1y 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 1 )h1`@%1 W42@F(P8"0H"@xH"`/PC1(" \=1 (f12!"P71@ڔ1 1+" 6h1`x""e11 ^"5x!1b" 1`M1[@1`P12X1`g$1 T11 11("`1?,191RD1`N2V"\!qa1d1pf"1 1]11.1`H_1""~(1$A1@2" @,AMh3!1AK1@̒1 "`01 fa82!2U!".#°"1@-2@Θ"!x11 6A@ Q !1"1`+22!719;1@<" 1`91`52 /!>(14111`1 F1`1r1`01@-" #`S !p#!H1@V4" 2 !P1@1+1@|2X 1 1"@"{1m11~# h(!Vc1Z2[U 0@P`p 0@P`p 0@P`p 0@P`p@ 0@P`p 0@P`p2x!r1։#!!12+!Ae111`*16r2ǻ!1LS111<1`1T1@(1w1 #` cX1ϩ2$!9=1@"x1`#`@ !(I1Bh1'1â1`1 12 !|1.2`?!7O1\1h1@^q1 x1}1%1@S"`1~1z1s1vk1 a1=V1 iI">" "X1 1 w"Q 1@"2"q0 1b1`!}1g1@wR1F="RP"H1@"x11 2@ٍ![k1J1,11@1E1 11 T121"`ո( 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p1 M1ρ1`111 1 K111" lXT1 "`z'1@"e6" '1L@11 &#eix'AN !h1`2!*S11d1 X2<!A_"@@ 191 j2I!1 1/1 F"1`p"p2!12S!ڽ21!X2U31"0"@111`2Ո!y1i1xW1D1B1111`v"1\12!."` 1W"v@1`e,1`2- 1@"`(1}2 `!E2.!m1 11@171J 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p"c1G1ӫ12=!/1k11@2k=!28b(K@ y1\" !"_61vǨ11 o1#!!j1V#`P2!_1 "6h$@K0"X1K"2p1`1@x11_61@R1`m1݅2`ʜ"ı!11%11ZR@`1 1 "#1`q1@1@11M"@p" " 1  "-@1L~"1 S1=1 '111 $11-1"H1m" 1@:B1-"`8" D 1O"`)"]P"1x " "1R1D2;!51 3 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p!" qP 1@c11`1 ;K"Ƞ81@11 h1@A1 K7!`]2! 2@\!i1 1@`1 "@>h71xd27W1 c1 '"` "[ 1AU(1 b|"#<"[#@;x !Q"8 1`A1" 112X!1&23!4>13F1K1N11Q$1` 1@P1"`XP1`11`{" X"{"`0."$1 (11"112@!01 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p1ͺ1`[11 y"K"1#а!i1|j23R1 Lk11 l1 w1" XT1`(1 1!@1j1e1`P"p "#`Y1F" "!1`h1*1@Q1w"`g1 11@1@? 1 <1DW1@o1~11 ש11i1@11 r"f#@" 81[111@1`y2e " 1,!1`^111@B1{1`#@!N1 741121@x1`~2 ʶ!1=1`u"@C4"F."# x!#(1T1`1n" 11s 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p1<Ř#WX"PO1 T1 W11 1<1`,1" *X11 PL@4/#6!Ũ"N@@N11:1@11 '"`?p1811 1@P#0y" !#ZH !(1W#`1`IY1@@w1`"@@"XWR@`!1 @S@ 2`!W111D 151H2v!&11ਲ1"`@1`Sm1S"1@"= 1@j"`^@1 =1(1 ?w1[1 m@1%2@ !mx"11@5"8" 9 "`X"`1P(1VC17"`-1&1 ,!#@9) 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p"`P>1`y1 1 n,1`o2 J@_!1 >1k1# ! SL@u1h?2ಞ!41@]"@kP1"@(+1 "Ø1K1`Jz"@u1`"2=@"hh1`H152!1'1`L#@C@ !181`11@X"1)2D9!E1O2U!W1`V1@R1K1 A1`41)%"`1~"1^" 1 1"֨"`n"1e 111`11 1@d"8H(1b.#@].F.1;1 W"x "zP"31@~ 2 .o!a"2`9L!D"@& 1;1: 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p"1Ƞ11`?1"+k1+1`I11@Y71;$BpdQ1 1-1q11L411 1"h11ǹ1@Y 1=" 1`R"@"("0" ""`"P01""1X~1 @"@^ 11 r81"1 <1`S1g1@x11 111@1 "1@1`~1mp1`t_1 L1x62@!1 d2!1ƕ"@( 1Y1` ;1 q1@"1`1"`1j"0"@5011111" x 1`2f1r1i1ta"`31`X1W 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p1@"@h 12S"-!1`I1`1T# q!z1 >1{1/" X 1l1`1g2,X-O@}"1`(+" f1m1.2`! "`1@2`4}!" 1`Π11/a1 PA1!1`"" @("(1P2}!0m"17"`11i"P?1 h"x 1 e"-2!""31w1t1s 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p11`x I@U2@>#1g1W!K@`$"q31@A"ːX1@1i1111@1`" P-1@cİ2!F1 x11`11"Rh5" '1@G1@214-R!Z"@1-1"x19+1P1r1`1 1G1@17ASTp!711`" " #0!E@1"2!m11`#f1G1@(1#@" !ݥ"01c"H(" 8("`1@"`(" "@"3"r.1801 "`C`"1`F1`111@"0#E1 "P#| 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p"@ 12۰31{1@1U(" 1e1@1@-11@ae11,1Q1a1@[11 '1ō"Y"!+16`161"11@=+1 j]2`=!1#8 !Q1 f2@Y">1 011Y1`"@?@ "j 2@!11@001@1`-L1S2T!Q1[J1 S>1G.1 "1p2`!S1M"`P"`1 &/1 "`"ĸ"91 "0("[(("f"!1P"`1`" #(1j" p 181!1 "41@_" #2 (("@41 #CH!k"z 0@P`p 0@P`p 0@P`p 0@P`p#`p 0@P`p 0@P`p1"`G3"e12!&A2L"R311S"11h1@O1."[Y1@2!#P1@" +"`f( "`7"X "8>1 ;2E!x""!X18"@1r" " 1-1.1Z1`n1P"#1"-19X"`82j"o|!2^"!^1d2`w"me!}O1 T6"z1 l11c11v"~0"1 2!"F19|"`1711h"2q!" #1f"`"("uK"U41@x1@N"`x "-11@R1"`h""  0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 2i!1O1 Ӣ11@/X1ॶ11@Vx11>2KЁ!n"E{1j<12 !Ot"12"CH"PZ" 4p "T1("@.X 181 11@+h1`^"!" 1 1 #" "`P 12@!82(!L1m1@1b1 1@11`811`1@1 ]1Á" 1@]G1@&1 " 111jr1L1 r'1!111`1q"92," !;`1`1@q11@Gz" J1J"xP"" &x1 "P91`11u" "8(1  0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p" X"@8Z1 j^1д1 2m!1-1 1]1@W1 $1=#1` 1s1jY"1*11131h""11 1<1q"P1a1X1` D"!1@a2`]!1`|B1_s2@F!2C!* "@=H 1@{1͞"x11 1FU@@0 1P1 10"1"@ 2 p![M"8"@"p 11""1@1F1 (2!1`u1@b"` "`c "`1 #@H"H"11 1 ir"X)"8K"`1 ʍ2R!z1@-*1Q]"`!2s!1! " 1{2nF`1g1 1~&1%81A2B!;1``,1 AT!1@N1ו1o1H2 "8!1" (1 V"@ 11U11@"@"89"@1 b "p1 FЈ11p1 1`j1U1@ZC1 31&1@o11@42 !U "$[" 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 1&11 y1@{1101(11`R1A@ L !116M#) d1`1@2Z!1 _1N"72!"` " 2T1Z"}C114P1jj"&"'"1?"X " 1& T@(?2`yo!A11 I2p!]>2`Z!,o1]z12{1q1`X_1E"" 1@`"@F`1t1c" @11s"@z1@֑1h1`@1 1(1d1 @(2 :!1Gp"<.1ֿ" 8h 11@v1a11P"UhJ"i(1p,"3"28"@XB"u((1(" 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`pA,0Ix!A!01e1`<11 1_1`[@(L@ $12`\!1-11@1`l#^h<12Ѝ`1@1@11g2@,!"d`1 AP1`px""2 !"@H!1[#y!"18$1W1`1`1%1h1`C2@kj!M2!g#@!1Ǝ1-p1fL1@B%"p 1@`p11`;z1XN1 "111@%1 w1`M1$1 1@"91&1@]g"`" 1 11@#12!1k1 Z1@L1WB"`0xa"V"813 12"" 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p1J81s11 2+FȀ`AKH "i!1`315A`DM!'h11@$:" hAO(!y1_`(P@^1L11 1 M9",h-1""1@1L1 1" H "x1 ^1e" }H-1 E2`5!"X 2e!P1 51d1@w11`e1"`2 <!<1ఔ1k1@@2`"x! 2s! ^"H("K1,11ց1 W#((""@c182 o@X1*1 ""@1`1"'"^0g1 c1V"H"">1@G"b"@&" #` 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p#_u`1|1@1@L11 !1q1?1:1#"3ЗA M!B1`άAO !2`!-1@c"x_1/1 "3"`1K1AR! S1 o111"0#@6)21 1 1A1w1`2"uH!8K1 }2W!~AV!" #"M(*x 1k1@T"%h "x1nƀ1x1`h1:1r 1 1 1@6"౨15"Y-2` !1@1:t1 AQ1/""9."` "P1@""x)" `P1_ ">1Z"@#   @P`p 0@P`p 0@P`p 0@P`p1@ Sp 0@P`p 0@P`p 1B"`L-#A! 2P"!529u"'!T2 !t1@GN@ӱ2`!B"C 3" ^PS"0Y#@u""E԰!< "V81y1ܰ1 @1@1 .W1@10"`"1@l1`ң#!1I1" N@2!"P2 X!12`!{b$V@pI1`;H1 ""@((1J01@1 {`1R/1`x"6(" k # 3X!@ ""`{1 1g1b28"!(",1"!`#1 T1d21@D11`(1"`%h[" bb1@{1n" 9m 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p1 BC11z1VA`RJXC!11v1 1N@1¦1"1[v1Z1@I1p1@k1`11D1#u"j!B1B "eX "+2!p" Z1zXH1 "`7"0N1@616n1"`'"5(>1 RK2`J""2&&H!Z\2 P!2j!22 Jg!Kd1A01(1h1 "_`121 ^1`А111q1jB"X11t"@ &1c191`10"(H#1y""@#@9@g! #@V1G"@1/"0" 0@P`p 0@P`p 0@P`p 0@P`p 0@P`p  ` ZP`LH@ DATA-END DATA-BEGIN <Ck qD  -{"7 0H91{13{- 5{*o v 19  f   ,!T qq, t)!q+ & p '3M$~<7'j_ 8++(1[H#:- rf#y2 B {(R4\4]3^2' W8W9%Ye (+ :l,m6l'lJ*m&] rl %m=%  i" J M$ h ! H3\ ?I5 \i J ( N)QF,)bE)2N)PG +)cD)N*K !';1'I$&s /R&% p.*M&2$I# -,x(   ! ( -oI , 9 `' ! J &%3-  *H S )% nA8FE +% $A F G (%  AGG *& @7H D )' 6I E )(0 V* 5  )+1"} G )+ l 3} W )- /$|G ). (l$5UX ).6}(*4U % 09U`) +l<\ $  1b=  + !F/4NH#  3"9 ;Y!  4#3O! 69 O=("k ()U} # 4)_ L  . 9EE&/T5z2;0VG%" AMr( l FW &RS * /4  3NS "' -QDc,  @ ,SD-" w+.6+\H-{* l>9(k"  BbO<C FNz  6l  z| 190 D 3 ` h -p<:4 DvD74   JDy (' =U  'Xg :01Q/+VD 41 *D68"6< Q 9E a G 4 $;   9 ^ G .y ^  e 1A7 Oa1,   ZN t}   ;P +" ` :O4 _E  #< FK  U  @ B pKy # 'D [ .$ AKk /' &n -l < JZ _ &!mlF ' ']8 #dW *[a  "gV +    a/i  ]\ + "![_4w Xd !5*  Wc 6   Vc 8   Se :   \^ E! Q \= "  NB [>" G U E # $2Q '  % Q? ' %0|   )# 1 S |Kl1F 6RAlG]6R=d& Z? 5O8lEK5TqP A \Q >CFh=]_$9 P DATA-END ATLAS-IO-END eckit-2.0.7/tests/geo/eckit_geo_cache/grid/orca/742e1f23882c68571cd820ecc3edce10.ek0000777000175000017500000000000015161702250040230 2d5bde4f52ff3a9bea5629cd9ac514410-742e1f23882c68571cd820ecc3edce10.ekustar alastairalastaireckit-2.0.7/tests/geo/eckit_geo_cache/grid/fesom/0000775000175000017500000000000015161702250022030 5ustar alastairalastair././@LongLink0000644000000000000000000000016600000000000011606 Lustar rootrooteckit-2.0.7/tests/geo/eckit_geo_cache/grid/fesom/e548b74fa53eef5ab412c6061330f043-1dbc5cbd2b726bed6161cf9c341adebd.ekeckit-2.0.7/tests/geo/eckit_geo_cache/grid/fesom/e548b74fa53eef5ab412c6061330f043-1dbc5cbd2b726bed610000664000175000017500000027101415161702250031115 0ustar alastairalastairATLAS-IO u`g(F: ryamlnone: METADATA-BEGIN { "version" : { "type" : "scalar", "datatype" : "uint64", "value" : 0, "base64" : "AAAAAAAAAAA=" }, "n" : { "type" : "scalar", "datatype" : "uint64", "value" : 5839, "base64" : "AAAAAAAAFs8=" }, "latitude" : { "type" : "array", "shape" : [5839], "datatype" : "real64", "data" : { "section" : 1 } }, "longitude" : { "type" : "array", "shape" : [5839], "datatype" : "real64", "data" : { "section" : 2 } } } METADATA-END INDEX-BEGIN |none:4none: INDEX-END DATA-BEGIN EזR@G}xR@K/eolR@gE@R@FR@-LYqR@TR@vR@d{YR@؆Z.R@C^R@S^R@c"Uz‚R@o[R@~KR@)_orR@+s`^R@;[tR@YykqgR@_zpRR@?'R@P5̠R@NUR@8,uR@ R@dGR@ɹ✁R@yR@R@[X7޻R@9OR@mdR@ٵR@幠R@0ەR@)j@R@%LR@R,QR@ߣݬ@R@`)eMR@97R@~yR@XUR@)_qR@r%@R@5\6 6R@6u4/5R@CnJR@$BR@ '=wOR@ 'R@҇ .R@X@NR@0^\R@[/{R@|a2R@ߪ8אR@6R@XRR@Q?'R@w/~R@=[R@{WD1AR@M2w0R@ R@ԻW1R@R@B`R@R@"xR@H4~R@/?R@$SR@OtsR@'bR@[fkR@ejR@:R@AJR@j#R@ ͥEQ@(ɄwQ@J^R@eJ`6R@X`TBR@{vcR@]y߇ZR@R@ "HR@u2!R@yY1R@.4R@ kR@w!0i$R@53vQ@3&R@uAj@"R@1ϰCR@̑:PR@L`o5R@Aڔe_R@ LYR@f gR@9E R@QQ@eR@LR@OaqR@طQR@UK:AR@ZR@POR@EW*uR@S[R@am R@IR@` gxR@ݷR@8 AR@cR@JAJR@V@1R@Cb :#R@:R@R@*Q@Ԗ[VR@!C5R@;)R@ PR@!Z|R@ S@ %㲵R@@PR@<~5R@0n'R@gR@p[\R@ CR@( iR@?_{R@MɤLR@efzR@?{5R@;\R@[`R@;R@xAdXHR@K0KR@M$9_.R@95^f&R@ިR@RTeQ@lhQ@KQ@EU R@cKQ@-rX(S@yZ-S@p 6oS@o~Mj%S@GTR@k,R@BR@tb* R@KP#0S@ S@-#fS@ds<oS@FORS@ RS@O90S@|IS@RS@z@S@U>_R@R@֣;R@ۻR@[27R@?RR@J+R@SԤR@79R@l,ޯR@i[1|R@J7 R@qpS@k|&R@0R@°֣R@6Q@\EiQ@[RbnQ@iQ@AşQ@+oʌQ@zQ@3'yQ@O QQ@*Q@ ?Q@FQ@EuQ@}׀VQ@UDVQ@ٚ1Q@ioVQ@vQ@$!sgQ@ pkQ@HhdQ@VQ@02Q@2Q@ָNQ@DQ@JfLQ@C>C_Q@,ܚQ@cqtQ@VQ@H۹Q@V BR@yz-R@g R@yR@?R@&R@繫DtR@8Q:IR@E[R@25R@w;R@K&R@'c!Q@HR@kQ@fQ@yQ@]򜓶Q@|5MR@^YR@2 -FS@˞SS@iwwfS@ ]wS@XYhS@_r/^S@_x@7S@3MS@3=S@"S@qo]5R@rS@*6\S@窝9S@fCS@_VsS@O>cS@- |S@StS@8ʄ nS@$FS@_!AbS@=XS@xCqS@9S@E5GS@"R@[X R@/ѿ S@tR@bߓ'R@-,gR@ 9S@X;s S@{JQ@@Ɛ`Q@8VzQ@V!i\Q@zAQ@dd&Q@TrTQ@O*SnQ@y>Q@(Q@<Q@O ӦQ@,Q@YD"]Q@NQ@T݅Q@eYuQ@s,vQ@9BuQ@ /fQ@ dQ@$PQ@ R@ߏӕR@ŸR@ePJR@42oR@>Q@.y}Q@;1R@.DR@gR@ײytJR@#‹/R@rR@[IQsR@,'2R@ņ:R@?eQR@$ .R@S@XݚnS@r@xS@BBS@++S@tنS@/!"Q@3-P@22 Q@YB Q@$P@5Q@i9P@yɺ߭P@Ul0P@G*P@枿Q@& Q@d~YqQ@E"P@#ʡ>P@xc6OQ@欜9Q@ Q@f4kQ@[ҦlXQ@jJ9Q@.Q@vq<Q@t Q@7-Q@|CQ@l;P@=mQ@ P*Q@˪X*Q@'DBPQ@S[ Q@eLfa*R@?R@_v ;R@ω*HR@vQ@ 21{Q@knQ@*XGQ@ߥX^G$R@ЖR@s2R@o5gTKQ@<7Z(Q@'1v6Q@^2qQ@uQ@\ӕQ@R0Q@\22Q@ Q@oUKQ@ "Q@%?Hk׿Q@(Q@egU4SQ@H4)RSQ@rUR@alR@cR@R@$R@5bR@_"2S@1rS@ nP!S@ \R@hwR@R@)R@LeR@h;dJR@CQ@e-BR@H'=S@LnS@&S@#S@ZGgS@3y=wS@n-S@߯vS@gxS@LڲӯS@-T@梌~S@^|S@|T@)wϸP@YP@z7Q,P@qZ}Q@Y"KuQ@ 4 Q@g!6OQ@]LP@d4U Q@leP@+P@kJP@};#SP@/<;17Q@Q@grQ@0nP@P* Q@I+ECP@eQ@[/ɷP@3R@㭼} R@g]oQ@O!Q@ 3Q@zQ@5_Q@sQ@0ȈQ@nIQ@s~}Q@U[t+3vQ@FQ@`ϕQ@^w[SQ@\23.Q@< :Q@_&YQ@}u`NMQ@w+C?Q@wQ@XR+uQ@fQ@],Q@%Nd`R@eR@eS@>R@ HGR@[$S@> DS@i3CiR@R@QұuR@ջ-Q@-)R@:N?R@ig,R@/(b{Q@CQ@,FQ@6^ebQ@(\S@࿒N'T@8 T@,T@T@S@j>XS@0}!xT@|>T@i^T@\P2T@ZX[T@1T@|DnJT@W9+S@[%*S@ې-S@S@ ZMS@XΨS@W4KP@ }t[P@*LyP@[ {P@0XSP@w.P@3/7~}Q@Gj:P@)Wm=P@\yP@-P@UP@qP)P@tFP@,eʞP@zuP@p4KP@ vP@C|:IP@mcpSP@'aP@p-OܸP@o]wP@l QkP@i&70P@P@vP@9P@L7P@ϢGP@1w]P@ :tP@jf)P@LZ&2P@P@ sP@d$AP@=*cP@-eP@!@P@eSQ@0&P@JKQ@G!`Q@ǣWkQ@?{Q@wQ@UQ@:gQ@LWQ@\e|Q@9]Q@e^*Q@DZ4@Q@nZP@0Q@ArfP@qzP@8 Q@EQ@~iOQ@2Q@'Q@8P@.P@hP@< P@;Q@@x P@P@ zP@av]aP@JR@w!S@ IS@`ԱS@(X^S@zHJS@/yR@ߘT#*R@ DDoR@廌OR@>w*mR@gp^R@=juR@Q@׋;R@JQ@ejpQ@q,zQ@b?S@AVzT!T@m|!sO?T@-2N&T@ `T@$Y> T@ކb<T@ȷl4 T@b}g,T@7~0@T@$ŠG[T@_IT@Qa]]eT@,oO@"SP@ۨKP@0T@|XÁT@5T@۠äT@1DT@TbT@dT@:T@fT@95T@-T@eP@?D8S@0XS@pVAdDS@ߠrS@i$S@դR@,KS@RʝR@gu5WR@jĹ3Q@|~D]@R@W,CQ@C|)"T@/iT@k6T@e֜ʹJT@T@L"T@ҏdT@ ҵDGT@5T@U5T@Z{T@pT@-A; T@g$VT@Yr0T@eT@W(T@/ T@!t(T@~T@iTT@T@MW|;T@%.T@-ׅT@F1T@tT@4@'CT@臘٬T@+g?T@^8nxT@DoT@!FT@P׾uT@9cT@9.[O@@Ɏ%O@aN@ N@qB(dvN@1͢dP@r{P@mbZm#R@/XR@P`rxR@i8Q@8jCͨT@lŢgT@0N2T@SUVT@aS@07U>T@usJ>T@\Q>T@/C8MT@ U@u`#U@MU@T@oST@GT@+T@KT@gm!T@HxKT@vT@LT@.#Eo%U@@K U@XE}1U@AbU@+7 T@Г= U@:9T@yT@}1h U@͘NT@=2~N@N@LŒN@YN@5bʰ?N@\c%M@bҬAN@q#9P@_GUP@RP@CKN,P@TP@#VVP@TuO&P@ P@82YP@lxWP@ {wllP@ZTL+P@X=P@cZ\P@bX1P@=o$P@CR6P@D;@|O@BRN@qN@[O@)%O@pV9.P@($@P@͛0P@Q P@USIO@0ّMO@91O@{!(P@x tO@?߄O@cd%O@KAIO@)L@Ϧ˯J@SK@E+cL@!0RO@BO@xuON@M O@2*O@aѾ͒O@EP@O@7LO@8PP@qѕyAO@[O O@ِtkO@ˮO@;S@oS@nS@d*T@;e ~S@6H'T@yEatS@%D_S@oJ4S@ERS@Ey^RR@زR@aGo+R@'%NR@4P̈4T@T@H#KT@iJ3@T@ET@; T@pU&wT@m:T@{OXL@mdΙ M@3P@2TO@$^8*O@huO@O@~O@| 2O@(T5O@X^]O@[;O@ YzO@xtO@4qO@07hO@myN@g5g~O@'9JEO@Mb0N@L5O@kELM@ocO@UH\tpN@c⋦O@ٗN@9|VO@WO@wS@_AS@嚟n.T@4喘S@"TgR@|6VWR@ٶ}R@D&*ec4S@P.=R@+bS@U?+qCR@}NQ@ =Q@}\`/8U@D"U@,0jU@iYYT@T])gZT@$Ndm$T@SU@%YrU@]AqU@J9s=U@bKU@]SU@l.|bU@WYTzU@#dAU@}U@%'gZU@I‚!U@O U@G"U@VU@4Bk:U@P5OU@緷_U@C2e*U@. yU@3=T@,'ۋT@C0T@`U=M@ċM@HgZ>M@sYaM@T L@PP',mM@wL@|Ht;O@wU|N@1O@s0N@[,QO@%W?O@Y>O@[/)O@l8dO@@{zO@\O@6O@藼xIO@+VO@W\N@$HN@L{M@,$M@ DCN@ @^M@ EGO@ EdN@ZN@`V1O@;FN@G^JO@p 5O@37hN@,OO=O@?N@ÞѱN@?IsB5N@ZվPM@3hRM@ًBL@uM@k3"rvN@!MN@LQSN@:QN@vM@6}4N@ix$t[N@90#N@𓛏O@[ /N@l iN@N@lTksT@ aT@uS@tZS@yTGS@?pS@|NS@Ax"6rS@$1SR@1.6)R@nטNS@wޫR@x'osU@_r4U@77$GU@kZT@OX~BU@ܺÌT@dT@ǯ.U@٣핍U@7U@M.cU@1fU@,0%U@m,cOU@ʭuU@j1 UU@߃M$7U@쇸U@ {5U@=U@EE+N(U@+R>U@Λ U@uT@cM`T@JT@a>V'T@w\T@!aL@k$~L@jHL@mp~L@jL@&: L@$nknK@%P=-L@9őlO@}.\O@9YFFO@4/E%O@iWN@H&ON@&yN@EI N@MN@N@NxN@eFe}O@@AQDO@w[N@q^O@ h M@qٽkM@ބ=M@%j/M@P`KN@t~=mM@Fc_M@7n(M@m/L@# O@+xO@;W-qN@ [`MmN@1QN@lvON@Y:"M@+©/N@+'QN@%!M@Hx xN@VbQaN@+L@>"M@t2M@d k,N@+BYN@4DN@SzpT@jhXT@2k6T@N. wS@*BS@!V8S@ujR@i؟PS@R@'6DR@ cU@+X>&U@&=U@'}U@PT@qK\U@,U@-Y=U@iIqV@JRU@8uU@\<rU@U@k=rU@S}7w߶U@M>U@T8#V@T@%8BT@u B̽S@@ĹS@PT@~T@0CbT@zhFU@?"U@ pU@sT@?nT@}|T@{T@ (T@p6T@O@k1S@2S@&JbS@ĐR@p>"S@h)HR@&b7L@PL@zK@H%{zJ@Q vJ@lisK[K@BʮO@H̆kN@x]N@M@S[dN@,,?AN@T N@gtPTN@N@ @ϣM@J;AN@\'N@tN@)yN@('N@x؉kL@'wLM@ߎRL@ɗtM@0C}M@ɓ'M@jgM@ ^L@q:L@G uN@wΖtM@7%N@5h2N@NzmM@KzNM@̨.^M@p *yN@.M@-Q,M@/@wM@M@oO?M@+M@7<4M@ ǫM@1M@ g U@D0T@rsT@RT@c;S@CS@S@XBgS@t4S@pfgS@AHJES@?QȒS@d2S@AS@ݳR@/CS@_?lR@ R@+QOR@Z[@R@N*3J@;)H@}H@MtI@kI@]T*K@pK@eK@t>J@_J@-ҚK@KPJ@3@1J@`N@3B0N@鮜#xM@MݘnM@M@ux6M@zM@$!M@M@#PM@x͊,BM@meLN@ L@/}PyL@F9L@;W07L@BY{K@kVNL@P9fM@ DM@집(L@!dL@%qM@.έJL@,J:L@)L@vYFM@-n|L@;q#L@#~eL@QgL@QM@- `JM@nT@T@ZZ]T@$T@9[T@U_TS@zS@;T@0S@9_lS@xZS@ A$V@6@@)vU@8d1l@U@eV@M"r`U@nU@U@< >U@x{nnU@9U@QaT@/E<&T@Z\S@pRT@!lXT@w֡T@qpsT@$"UU@ٜ. T@-۰S@<~S@AoS@xn)fS@^zqS@ͬt"S@. S@Po_tAS@>_S@(2H'R@ đR@'S4R@wBR@GrR@K^̠R@K 5I@)мaI@O~ ZJ@hI@[PI@O=H@Ɗ㉔H@4TԙH@UJ@ Z1L@73L@_߲K@W(K@mK@< ;L@CplL@%bAzL@ sϞK@&K@eeK@fǃrK@`jCL@E>3xL@Ć#L@ƷK@f=L\K@{H|@L@QZL@<}rE@H4E@"BfE@'oIE@9.ZD@xdD@'T@K1 ȰT@~BmT@T@ϑfT@h7#T@EKT@+x. T@4S@BMjdS@eS@Рc_U@sB@U@}8U@x%*U@Qtk;'U@aSU@;Vg 1U@BfeeU@U@# T@h0Tg3U@bQT@zT@'RtjH@\F@eE@P F@yG@MG@ݷ<AF@L3G@i@H@v I@99 I@Ӛ J@2#J@ :|-ÞJ@@'E;@ @9@@1D99@ҕ7@7ev/7@-/¸7@ %h5@Cxm6T@YUS@0S@)N^T@Co-T@P'bS@ݮqS@/cL2S@ $ЃS@kS@c.aS@ff[ .S@ iR@rQS@7R@~'ϹR@#/kbT@>@G]T@Zх%T@˵ _S@qS@ AbS@-FT@I4jT@Y;T~T@QVa T@t$JS@us:T@G pS@ÿXDS@I蝓S@YWS@r$vR@|=fR@;xU[R@|2oR@bʻ_R@D@RE@kG@ụH@ pCF I@41CF@TGjG@[uF@oqF@8)E@G@$t3I@@N>G@ eR I@~Sc I@ ?I@L$H@:J@2I@yaI@Mp}OH@qI@@? I@·I@<׆tJ@XmH@jH@$-@I@qI@&[03I@w1D@D@C@AgҦjyC@@Kxvf>@Z[B@6Q3B@e>B@ۇ@@͖@@d=@^<;@dt;@N5A9@~6@G.j6@:kr8@7@k)Ah3@D|S@JEES@~U+S@/%S@PS@}x T@fPT@N&S@PS@ mLS@qcS@k-S@4a(S@[eN{S@kRS@_S@7eȸR@AqR@`cݍR@;>OR@&@R@q qQ@ɾJ8R@5vghR@&rt#D@X0D@>Ӑ3E@+nI E@Z3C@UH@yARH@jV@G@#`\F@Uj$YH@uLQF@E@@p:L=@3s;@LT}>@3:@W;?@10R:8@vb:@#ܓ6@_PH/9@'uL6@<۰4@p24@g7@PJ2@=8p5@`5ؙ3@e6p2.6@EfdS@ZexS@)S@ʤR@OS@>'R@b<9_S@c 4BS@V S@<5S@#xhS@4 ȀS@!cvSS@R@y!R@f bR@MƚS@q-mK.R@IBR@-TXR@7S@މR@Ѡ_KR@,Q@D׷Q@]vC@ZuD@ uD@YECB@-gmzG@th[`qF@!|mVE@xb~G@CSOD@|GhuE@XJ C@3OC@A]TAE@g`F@ty&D@G@OE@f24G@0=ףG@[zH@)dԢ+G@ЮG@=SG@2uc'G@h~H@tͭYF@'Š~F@{G@lgA@/KA@u[OB@ItpE@@<@@@ :@M:@cT <@)?@=E :@W6/}8@Mڙ8@$(2@Y+:1@J=0S@[-$S@8yJ*R@R@MW GkR@G R@ϤR@@yR@_JшR@mhjR@AR@/ &R@A\?R@apR@!R@O<%oTR@-Uk2R@uR@7kR@7R@"UB@ϱB@ryA@5SsE@~_~F@HD@@dF@HbhE@{ƒ| 0F@hmE@gDE@_B@uSC@ܾ~E@)RJ F@5nԇE@J4ffF@7`F@n,F@M!PG@+F@WN=@MZ?@`)`<@I?@#KA@@aٚ ;@sD7*U<@x=;@Rm"`5@뷔Y7@h7@ zKX:@7+6@c& 8@4D]8@ȍ8@\s4@h`2@}DN5@f5@Iq;4@#z1@|g}oM3@.8A3@͈''@)0@Y?+-@]إR@ #%R@E}Q@NR@'gTR@`+!R@tQ@ ԕi~R@Q@0_Q@?cmfQR@38!4R@cR@wthQ@2c R@t`uQ@D>@D04>@} @@E@H$:D@.`A@,8B@D2vT@@yI+D@'ˆWB@饟TD@,W1 {C@y3lA@"D@38cD@|D@1uAC@߹eD@@D@ :@De7@Z<8@hDz)9@C=UV4@m׶6@`y;;4@Sc6@}8k8@].6@%h*64@,ڍT1@Ld2@I)72@͸(2,1@Ê2@P -@CMLQ*@TC40@@R@[lQ@-wD R@phŻQ@]M|LQ@wnվQ@';R[Q@@WQ@K7YQ@sʢ;Q@ ?8@@nM?@L D@i;# B@\ A@yݍY@C@XBC@]⼄A@TQiB@Yv*ݏA@V՗B@A&C@RnC@cnv#<@u8@߳I<@P@:7@oF}N4@HCM5@SjK(>3@ x3@;4)0@]2@b.@i43@|@.@Oi:p+@*2@*@s_(@Q@XQ@#|P@-P@;+YP@c]U%P@NeO@ҥHQ@ay@>@=e$<@C0:@e; :@Vp@@eN:>@+'`C@@O),@@$9$B@˩@@xVA@u ZC@9u3aB@ge6@JH_I5@'k6O3@.z/@KXU31@{J{TG*@&0@iLdF/@R'@!".@[81*@*V/@)R(@G0@XX/@0yEN[-@<5P@sc K@XmJ@O@[BN@~(6N@ۢO@U:.M@゚L@J5@Ϗ5@6|3@1"F9@[@@7@ϙ Z7@ݸ O:@u}!}A@S>@@~<@.D>@xZ?@L0<@02@@qA@Qyh?@J-@@hb\@@A@rw#4@E-@ɸ2@?A8N0@F5L9Ъ0@&@\3O1(@h[`%@ RH9"@AR462*@Caw*@Û*@x-|f%@;i3@'L@3 / jK@Kn~J@;tQ,K@fNH@+ ZҶI@9HUN@h|N@+r 6N@;{M@Y(jM@ >5N@wM@KK@߶K@vaL@OFK@b!BQL@5ֆ2@uc-.@hE)3@q0u4@Cͩ6@ɠC?9@s't:@fS<@pa"~;@8! A>@8@+H?@݊)nA(@+X M-@Ri.@:O #@&1f@d'@>@[z @HӲ#@e#MCM@P/|J@ [7!@D#@ l(0!@Wn@ǃ"@M5 @E*@u(&@A @)@i,'10@aN@{LL@W4 M@D$K@ӣCH@5s=H@[gI@a@˳J@x@s@I@Y[ nH@1ñF@(L@oPM@]NK@@lWVVL@#L@I?q{K@tkMXM@^6L@HCL@gdaL@IJJ@C՚xK@HA%eJ@P٢K@:Ss/@c I0@}T0@5Ѫu8@zC5@jUh:@L^#@Őp6@6@,@?d3_a@uO,? ށ@ O>'*@q%@@RJ@]A\K@ K@ZzL@@>L@yI@H@&G@ӚH@˪ I5I@s2HE@XxXC@;P@J@XJ@9܄WJ@#YV5K@J@CK@sA^BH@6J@D͉ J@#}PI@wor*J@p8I@)Q`J@iM@c5,@`Hą&@&ˤ'@o>3@[J@(@'X @g5@e/?s}qq  ?Mz\ @m @५"@r"@)9@=9@$U@%1J@9՜J@D_Q.K@(I@/?J@'jeK@V}MJ@@riH@(aH@SH@~bE@!Ã+E@Q+}F@Y`H@lgD@9[XD@ @^oD@ǥNd E@z[F@F@F@W-G@+FKB@K D@i~A@ݻ(;@@CYB@px@@dvTr>@:t;3@*p#{"2@ lcc%@|}ٳn .@RBy@\? S@v| @H>:@㯠?k]ˋ%@TVO @} H@ӖH@%$<xH@r+YDJ@K}P7I@K]I@0'O"I@˂H@PyVZG@i&H@%J!@惯@s @)@0}ؿcv @Lgiu6@OTS~ ?Ʌz`P$i{Ϸ@C1(ĿrW @bm@J/M?-)=m@UE%H@i1G@+n?9GI@1H)QK@g4J@UYK@UI@x&,G@ ua7I@"ن I@QH@qZ=G@G@U=֗G@/,G@ᔂ7hF@-cF@EC@cǃQB@IE@.>ZF@䍹X>E@G9D@,F NE@ D@B@ SZE@eNhC@N;|B@Ϥ,T>@bs A@^ ^>@56@%ڥ<@->@ڕ;@5ע:@d+6@ѩL#/@|js;/@ߎ92@¦.@𪧊8Y,@E/ҩ:&@&@%y(*@D_)@13% @\QҝNH@šiE@{_VcE@a>6MF@G@O܉F@!zC@0]LE@ 3E@B@,[B@~#PA@Ko#D@lC@4;A@VBA@Bz!@@S`7;`+"Z0h6y@P@@wJ3A@u~腌:@d:@+Q&88@B`{6@ek<@uO:@r xj8@ ):@i 8@Aߌ2@D[tJ4@"OfF0@{ m*@"1@cxTF2@o/*@(5ug'@zéLJ"@e|"l*@1Aev#@$]o@XG&$@A: @Ѻx!~%<|Â+%n3yX =ƕ@l>l=\Šw0wVP&q. M @@4H,D@k F@=H8E@WwD@G3C@6Zi-?F@YhkmYC@u-\B@G;kD@ybA@81w҅cq ]^. @Spa?Re7@`.'\F6waI"Jb6a5oTY㴻$D^ %_;Bu9)&5i1+#,e%|-E$lw~ >u;-U< sCheo| =C BA_pD@;OkqC@$&:] E@ʱF@&%E@txD@HND@@W|gؿB@jx?@">@wR+3Ad2Kܮ9{1TKN+(n;lz/B7(%Bi>y|;TUh eE?^ DŁBI{_B,ߦ @"%J56L_ #KC 3tMxO:Lx:N,)fP}PO8]O"k %ZP;P O@{8@~=@\\<@˺5@,X8@ȣZr6@MNXbl4@ێuV3@}S-TL&@$.q5k/@6FjY(%@](#*@P\k@y#@\E_@G"N@ꐐ>@ Uw@rE*1rW0ck10m $ 8SH7R#)j#Bq@ztZn&-I -5SI@m&:?Zk9M8kUC@M&|A@ cC@kM@@X޸C@#>@H}eA@/S;@~"&'ch'"c!| %g}V̌s<3La|zw&@TG? >ԝMeN!gtN[ՋL/_:J񫙁;F_ipSbE4eG<[ "H(b4I3 edISV6 ,zE b#Fo+ 8{+q+-Ȳѿ#V;(}cI8V63 G]6G64T{[M2,a0$!q=["])Գ+:#&F;osqjK;xo=eJt<73C4?dDEbfEdJE1QC M@h&Bu}o?-ݏ1A@y]M?@R$P1A@CB@j"gpB@RAC@ fwA@‚>B@-?@aXa>A@mi6@@/;8>@0;@q[7@ p?@=-@6<@h A7@ܽԶ4Hƍ^8Ax 3n5q&5϶N2p<;; _9,0xGѸ9z=lM[@xk֠D+0Bxv@-MB` CH K=lYMwЬL·QrNG'tM׾>ANݪҳO-ybPU+`2@S̺}6@ g˫9@U[@Icu7@ՠ ;@y5S43@o$h)X&q0& YKvo(N[`EAiע![>ؚJl"(3mCpQlMQl)&N })GN(;]My=ȏOLlyF^99KHءKgsuBJm0GLRzG/ |/HөaHi\J-'KW.|K2}S2x"0`P'Wp4-diۿ+#2~8mK)F)9>4 m7J+`^/1)09/| @5??%I=C&AhX'Ds; D{V+@VPAbfoB0MBtA@A2X?@V?@- B<@]Y$=@Ȏ Y@@g?:@hg_4@@CК?@7(NW:@}z;@ɧ>@?>==1@Mj'3@;N<.@q)*FQ}4 Q%^ugF@E!@9SHVF[Ȁ=9:PCA0鉒DC{%D ;B㴷ƭ3CѾ[ A[UI{wIDAI;2JOI;H'6lZ9Oԛ13.@TJ$%'@q(@O6iC0@~+@@r$@68@m@2b@TL42@hu0j4%#ٛ1 {0>&*,2o--m 2@:V8@;@9k28@UaE5@%4@I=6@]*4@j -5O6)I]}05ZHt!=BˇD9M˟d1MBLO'LoB,IjLS}>K06eqK_ӊNCNeWOC72FGH^IGr(0y"R`3쎲kh3^H5_16X763Ne\1GVp3L%7+p/960cmW39 \Q;^ 853 ?83w=XѶ=b #DKSEG ?DY|IFBE!<@;-R;>@[i9@oY&<@$kC8@EoT7@16@v9@{8@ې\<@:@w+7@Cֻ(:@"};@ A6@S3@Z2@۱"-@MNiԮ)+@R7KX,@^Fz4@/ƌap)@_<:!@Xh{,@s][@@,[ic<_X?JFm`FG{$Fn*DJE!%z?%@!yC-בDD}GJ]tRG/K] UKflUnK|#ЛLscL=EGNE2Naޖ6OIgvLUli1PݔJ+LPNgPW]q!QcڢQHvPm\OJPm@IM7GJ18 K5ݕP`VJ P WcOQ; P ъlNroR[PO&/Q,KP|aP_zFx@ҴH4I)=I!۴ 7C:_>/6P5WG8,.U!;S~R7Qj8K;wt9`M=m0iy;[GA@lwQ;CUx &%@(@a u(@uo @;;د<^{?o1XADGnP@m,y@hu-?ZEESB kB?C+GmdVD;\ 0@P2@{&5@3z(P5@L!2@3U(p3@Wg 1@dfZ.@/)2@D&@s3`/@G+@%X0@פ+@(<w)@|D0@ ~@ 3.̨#@</?u,m@y_0@tWq@&;T5@hj_z?/{w!A@I)?;h@2NOGJi!CL>@4R? 7D;WB_|AdDA;/WoEF#?HEQfGsL,63@<"Sv3@L z0@0ؐ|+@ͷA~0@m1#2@h})@ّ1@(U -@KЯGB1@ey9'/@pY*@|J?@*@ԡDa} @4iK(@s?"_2#?M4!@Y멩s@M_K{|] P R`xA`HSL]LPO\|NlߠKN'%b@Oj7DRlx-/RɆlQ-vQtZQ- qP(=ʽ+P=ٹQyEQ'mTϴRSDSW)ŔRyUBRzmR\0. B!%AoO5CscR.AhqCC[AenBlџ,J譧CHIKԝ=R$ F@)bGڂD~'w%@a3Cs,@că-@L)@b[(@Yi~r#@9+@0'@^#@M} #@ @T2@"@[,i@R&@U=#@M 톲@0d&i@=_@{ֿ %'h6C?a[gMg0a=]'i4%zb#3!D; pjt8jn;eeV1;䮹?WzQ?RY?PQQP}QI΂Qϵ7QZ]QeqLtP;X4PL~YP}rJ"M[#I-Lu G^JHOONX::M\!O cF֋sFi:_TAgO@ַзRa1߀RXfO,RS'1S%26R,\$=RRPiLS2CRbFRPskxSx"F܍D0MjTA+5+Ȉ.Cr4 FA^8sW>9!f<8{>q{zS.^=R{m?O#R@mDT@mR͏"S2R2Rz?fSSH! RmBŠRiE=AR)s%R#RpQR<˫RXQtR6pRP+9?Y^X @B@hr!@8Ln)@$2I&@ &!@DM614@SR3@]:q,@o//@8G}tH(@'f%@v;x0@i҈\R ɴ %u+u$E$yտҖƛ,%z([t7ڬ1@W6e€-@-:&@qJ 1qUYAkSUOMJP>k =POB:PeSP9ѽ7Qv^H[Rs d1R)qCQ/:{Q,̶jQ/hye-S*xuRڻ-S@ @T+5=@e8<1@ :?Hclf?y.>S/SpI&(H3,X)ZRm+kyj`Fw8!uJP"DPD!-" AheKBٙSA집(Q<'jRWHQ&׃QQm+i}yP%@5@@%IF==lAv-6IPچ^K%*Ib^IfDKpL ؟N3,qcDOGDD]>HaHH}mSCh?Gj6B 78F RC?hs[UB?Zt4Q@rq URS϶RjS=uROd]RpR!RUnURҏRsj*Rr*RlacRhW\RH݆R ΎRJ.RUpRL+1#J3+5v^ѹ8dX=l^I>tt:ع@2@6*LAڪR$p]R'9VRYbRccwR#XR!yќRblR w]RT0f+RX2WR;קRzyR- -V/w1G9vew@ג@ {Z@' Z@۬`pإ%@d~@[\%5%@_:T @s&(l .@鸬TD7O Od n1'X|C,`mtC'E@$@9_&,@%gg!)@] (@#oz"@s'[#@dm[vu@-dk$ @r_@Q[P3kߝ,y60bTCT6BpzjDlD{BE#EO0/E@4]CnrP-VP06CP PdR5WQ%JQ4R-OR$ZGrRtQ=Rه|`Ea{pDva&N\f8jN*meI1/HHGBUxGW3oK0nJA?<@]13?ӱ]@w?yFdK@uP@)Jl @Ma!?TW?56f?19y3eTu3Njü" 0!F0S qo11* 5(|gm&'r&M}-#uH(3;BkXשCQ_9oQ8Rl{?QoQCMQgFP؞Q<HQiEP d hO'yP!E+"D\pS(IANM$N#iNOSMptg;KpƱv\GO ڻF4 D }+C»rxB|R,R;sZrR~dBRJ~CRocNgQͷNRxgR>"pR SR~.Rԝ?RWRRE3 4g)YN!69ഐBO:AY̲A̩BDe9yB&^-D BiBI7xeD7!xDrD=$CqaICx`RG6vR#o숔R)\Rq@U @KVBg@(e1 @ Q@ _A@kDk?:'8i-]P9N9|i;n<+V6At^n=tIaBnvAR FSAF͕1P_UOaCmO<OJ?P{ P92/O/`|PSɸwQ9I QIoxQAvQ-_`DQCRW@Q=N[`I-VH[|MSt M 2M5\.q?:o5?ݕd`%y^78r06Oy4sA5+ҍ2Mn<8.|!/  )8/-p*ݼC%x>E;@Pk%Ql!lkQUkrQ˴4rPYYP ߘxxFhgG6RD`FeF9(uJ[q0_LҹrN@&ِWNsh+.Lh*GxOUI n4GQٱDB0EFT mD88YQM5Q?7lRF. =R1 2QXtqQt0RiJaRI PRMqŎ$ROQXR>!Q=t(*R}7_ARKLK/8c=˜ @4?!6Ƽ@A]s^@Oi8@(`nMA1'<B|M[BEجREPq5`R#ȑh-|R9GsaR6bR]?RU퓫RrޞRSE^Rf.rR&+xRޘ>o2 G24F[IUG?~ w @Q .H,?t#\@_@b 5*U/-&y~!WLm  iB&uS΄)Rcz.|N)0Ūm?mQE&W?\9@X;%@/o @ E?8mJH ]J3IEf%GirHQ8(Q,MuQ> QGŇ RK_-Q ?RȾ> 􊑩@i _ CzBSr?A7˽B'=xByl]FcSECk"ES\FgZE{@QleRY|g6RR)y-RQ-NQ1R?֚rRNU~R=wMR7Ե9&uي9?ح;m<3dA8c64O 5SЧ!??a! .CţMU?vo=;U@˻M,&#ýVL+d/!e115?Tu3Xp2{@6e=3gU7Ɔ }D lg#_X*%ڿM1,F@L;o-}ò?\kG$c AlT쿗Gso]!j >=SMk[==V`&Sb@Hb=$?;[%F]HYmOHs+,@|LgA7P ?nB D("< =Ek l(PEj`CЈCLLFtk G !*IFP9vMP(HOT0`WQJT(QGDhP2P> Q܄PboyKgnMqêYyN$j!=M?O`'MNs #ۡ*ox')I'wF 6-#8$oГI7GH㠂%{y2C<(<4}1(9](H7̭]V;B6)@ G{?PrŐP{P'$QX5P)NO`+Lq@Mc OBnN ?XH:K-K׏!JeHԘQ%JUsIq$H `&Q= Q"NQ6,m=`Q;kP낏Q5 S$(QhQyQ(Q8Pq[H>Qa%!Q_>E&@3@Ta> o?q=EUE+#FВBkArf`@XB$AhBHC}`COQՉzG}c>mG;2@F8`FFQ=Q{NQ oQ{`KQzcQR$}uQ@7{QYVfrQFFQP &@QwkQ*Pش@DQ pWFQ?Q0NtDA^EAAI@3?A69 `B$T8B%K0AFyFuq ^F]BL+CXICZpa?B|MB?3daDf7EDͨP2E;ZHT{/iH%^'QЂTQ][Q+PGuQ%%FQȝr@QdzkQ@ ?s5"< ;pyA: @'bS>xP-:D;k"UJ(K=:L5)v&JoZ4"J\*P]>P0—%oPj_tOQPO%oRPWzMPA>PoYP¡PK8kQ=|PQń$TkCC1Ac2`CA bNB/?Z9ZAn(\>3tmS11? h6"*|7 p: q.i*Q1 <+4o4Լp.sp&v386p5>5A֊mCβBcrWGOH\2-!H jJ=6E!|5FVH8Ei߻QTKt_K7ʚJ>qMw NPD$8$PꢰPv3Ku5M7G3` 7o<6tC5u86Q1$(h ChJ(mI?s<PuwQNGB|cKA#%/IKo)9Lk.PڧP{}`PrŵP5LP.bLPP;?P^ P*PM!Dٗ4[|9);(շ8hs2}H7P{B]A98: 8 bMa<ɦBN\ߝMe Mi)PbT{P)lPxiPP 6!dPCߋEC6 E.ғHhkIh2H)VHe^)xH DFH%F $_Fi"k#E9/Ee}pF/qѶ2GPqG  H3S{4JW#JjCQ]KL@?Q)QYLPLx-QW@Q췉mBUf/CLB`KX>=Ҽ@a\=MxH??G ݥ8OB-<=F'?t@"!hEU@{>g4HcDRCHtE wF- JW|KCM5LA8TQI0\L-!lM}MUN OQ[uNM O?$_*P [Om:pO3=PG&@#@=QiKn@?_AMXC癚Bv9Rn=HX?B+13Y@ m@~/A##AGL[JmjKZ2UI9JcȮ ][M'oN 8N!piObTN9F, PuҮP ON~}rPS^PN`k,P9CkP)]A.!BԠJCBb+&>ACt>AS*A/kA<Ʈ@QC CT4mC8_6KnDHD{gDe_E㡔rE``EXh1%FmK9z񄮛LG(BKy ;iN= NubkLKF;wP;KSIPMO'P%DOOjxO `EPXMFjG%B9P(BDv5B}`"9tDE9Dר:)B3`XzA,IA/m"xBS{CwvB]e>C6rICqMB[mD*?D ]rE۽D63D4:3D%ұEYNFOF^qFB `kGFWuLGM,WOKALd Nc]kM{AvNS4MlHODA&W2PES4O;GC O9:PwwO RPn{OqHpfQFHFGeGp'HWKs JJ! IM=IKtIBIP(PthPcAfP1.nPx]NEyH`N^,)M32͆O[XA=bP-P}&BOG(4PG Q{PixPpfPu-PGPC 3.L͚1KDm8DzM%>D1J#(DUoE*EI}E&ɢNFO>rFoD9h)F'6_E*F?F _FyE`GEJ~FqGڀHG+.?H6aNcH}mlI'5HL*.ANUyNӣrNؓ (eO%HGcO3:8Ol0Ώ%ORI3+rJ*5OsJe pI5ũPJsuK2=LsXm@KyL݆$lM^bUM3(HPdioP9"ߙP圀s8O';Pu\Pܛ,PdhP&OaPY'mP;mQeP=PdP=Pas#H(J7tJ+lHI̳PtJ<ӘFH*J,GpNMHMtvOg2*O~ăwNs,kr\OOHKObNP9!ɷP[* .PևjPD#MBXM1IhsiL`t&_NN}hF ZM,>KF';E%TE]FP!tGjfhG13Ff7iYG5Z9HexnHm|.F/OF2G zHlG)MIxDoJ`߷7JOtH(I|H |7t'NDۜeN#Gk?O0ߟ*JI ý9K gCMJEɯL)LBM|%N լJOuHeLOUtlP0N PgO{v=P H(Pkܿ:PɠQhPhPw?PAKP xCPUs_&PgGP&[Je"c J3H(O΀HאKSC6~K8l?LM [>MDVLN*oDOe Og: PFsOD|PՍSJPَOsPϥP+׺XWPj>yNleG^_uH+Ɋ{ IwrIHq{)H/`sH>wIy2J Q^HT=~I5 XJo lrI[MN_zNZT9K 'N7KSR2LNeL:krXlOlBa]OCOEBP٢7\P$ PáHP}opP['.PhPئO4P\PtǀMP\tP6>SP[NPȠǫPҋJPh`tPvJt :L! QLZY%MMҮMgFHNP($NO?.OLO|lONp츹OЬWO7퀧5PS<-gPPHO`*UNmIK(mJjLJQ IdRf^K53.SIқJ/?2HJL!FKr|J`_LJhK~SK0O|LP^g MoM09M xuNMxKFXNH+O9O琱OI P 9Pב/Ok݅7P)TXPebPD9q-Py GPZ* ݞP:),hP`ϣSK2Kk M15wM7NbJMGDFN?^pO}]EN$|$ANnWOtgOUaWO1s~PP >P.\OxK$OfP%KYAKuX~lMt3K1EKoL((qXLU,lL_LEcLE !CMhMXXANu 'NTjO>8ZwO=OUBOdOG"%P=07(P]F4Pr_PAomAP4)P0H1Pn2RP=o=L`FONhWM:9D0Nm0 O_RqOh0PO-+q"PG]P_3Phl@@PLt`P #PBPu֭PՁLPOtPD!dnPmJFPJpPNqOlOmO]+X3P0:qP_GO|tPDOQF/Pd& P[ϗ]PhM'P𿶺oP_IXPP DATA-END DATA-BEGIN ARXr@H!Dr@"r@ѹ!kr@lwr@Hr@DuΪr@f'r@|ur@,W^r@Q8|r@0r@X"kxr@Խ;?Rr@sr@%tor@0LBr@?Tr@ -}zr@W1r@lr@;ۣr@+r@bek3r@Yr@5 `r@3>r@C$gqr@8զr@}V2r@ drr@Lr@;֙ r@Ver@(.tr@r<ұr@Ir@3=r@(xx=&r@Ka9r@@H>r@4r@&=r@Yr@J7r@/r@.Vr@9_r@]r@Qacr@wSr@iǮr@ r@ ?.r@Tr@3:;r@&br@4r@Tr@ͣr@qnJ0]u@+=Uu@ r@W}xq@ fEr@bHq@ r@#Ъq@D,iq@Væ"r@ߤ|u r@+,r@>vc>r@5Cr@Z93Ar@r@Jvsu@ygu@yR"[u@Z[]u@0_j]u@UhFu@LʓCu@iG=nu@q@AKu@ "^u@6u@mu@qg9nu@(u@a u@Xç ([ @YԘQHOu@nu@4-C.#v@]nu@Bu@%;Du@Ru@{Ϧu@Zyu@F|u@ k|u@;U!Yu@0f7RUu@O6/nu@y(hu@7y~@A@0@_tu@t5u@+u@d 1u@WWu@˅0v@)9v@x.Bgu@)+s'v@"#V5m@?@*)I#D @KeKu@Řsu@u@uɏu@yu@=f2Ju@V>?qu@%=(lu@EZx_u@gu@3p#u@vޏq@i~7q@qjr@hPq@m_?q@Ln\uq@FZ r@ۻt] r@Or@)7r@r(ȑr@fUqPr@ܲ r@asgr@8`]r@:X.r@+hr@P?r@8̓Cu@p'5*@T/@=21]*@Ӏ_¶+@+)@^0@tS)@(J/W,)@ӟƆ7 @&>#@Ev@*5{`!v@j4J A>v@rǿL:\= @+@_{q@@%o@sM܅A$@Uu@6tv@u@tu@4 bu@uنiu@eEUTu@oJu@8su@32u@WRPq@eLq@SfFq@ aq@'C1Ur@`gq@7ofq@6t r@r@oOr@R]r@ϭ꟮r@_ާr@̞pr@55r@ )r@Par@WC\r@r@~r@r@n,_r@g)\r@3` s@|+s@X2)t@^<>t@?Û56u@eSu@/8u@6/u@k!u@d;u@C1u@d;t@ e+yLu@׻ju@IGu@!uu@oG u@C`I2@x!T-@`_}ob-@C(V0@{K'@C"#B+@d$%@7"5@¹!@ 뵏ˠ7@54@\ﶊ@. W@ u@g#Eu@H=?V@-@Y7Mv@ ^>v@H*v@)<#v@` 6u@u@!Zv@(u@!b98u@$G4@0#5X1@O2n2@=[u0@#Q.@`-@+\.@e>|'@Z2@4@7Mx3@/X[7@o`D:@50:X6@v`:@];fi9@e8@(5 "@m7,v@M(*?1T3I@ɡ???[@T"@:+@]>$@[Rev@-lu@p㶸u@eyu@d{Pu@"/q@e:=Dq@Gr@B`,r@l5'r@ Mߪr@op6r@K#r@2!r@gv+r@X53r@J|I^r@)(#r@Q ͈s@Q3Cr@t3[1r@9@~s@cSZt@ אt@O\Et@xcAu@71t@QTeu@s u@ 3u@mu@}u@s&uu@( t@?xlct@)t@wC-อu@KNu@z׹u@&|^u@-Zu@Ynu@,6zu@%k{u@m[R5@Q0@s )"@zaEL0@=ho^'@er4@+'PVF@qw 1 @Pz"@8@MD>@wSI @J\@mzm1ȿTѠͶu@Qu@jyi3Y;v@0v@*:a!v@4du@K4uu@̿!u@`G$u@_$Y,7@RR:2@7~K7@8 !|6@-x3@+4@!70@ m6@B(;@;@k<@E3.r<@H[>@mT2z@@f>@lgS?@+8A@8pA@AFROv@kC.v@3x܄? 7p@@mYg|!@}^( @G^u@uB v@,u@ř@u@Lͫu@Hu@+`u@D6r@qY@q@4Rn'r@Voq@nq@T0q@O8r@r@sRt@U@t@Kt@;:t@'|t@qt@8t@6t@t@ט:r@ox/r@}A/Qr@fr@B{r@YIb{r@7?r@*L)Br@der@dŧWs@_H=r@NSs@s@O4u@S/u@# (u@;Cu@iڱ@u@wt@"7)u@Fu@6t@/ot@;sԑt@M5Eu@ @pYA@yeֈA@@sBn>@L-,B@A%c@@KJB@'լBB@B@C@t-ٜ'@]t¬*@GtRLE'ˌ&@#@p`ա?hZH4]+}@Fw٘@O[@v@`u@1 ݀u@Nsv@E4.u@֙Mr@7M)r@|jnq@q@cLr@v]*r@)u?sr@ӆ1Wur@cr@g◲t@m}~vt@8st@D%t@w17t@~t@3-hlt@i^o}t@8)F]t@FHNr@ܠsr@QKr@Y%~Or@\r@g%r@<8bs@Ln s@/IOt@-#t@Sܲu@km>du@?Cu@st@usu@Ìu@[k9u@jv>@38;@V:@Ao :@ii@@̷b?@HgC@@MA q=@=W7zA@H@@2u{u@{F'C2@mBG0@%b7v@J 9P(@y>"@@E?,< @stKa@޷k#@CX-++`Bt@>@'r B@C@4ZCE*D@!̷D@VR8bF@_MC@AqF@b@E@E[:G@h":T<@*;@cdA@3O=@jtODu@j1 t@V}w/@ l8 6@^q^0@?(p3]4@Cg929<@A> @V-!v@cNYv@H];u@Z(bڅu@%qDZu@ T0u@#Ҁ?r@ęKs@H Bfs@c2Ls@= ds@k1s@[<ͺr@o/#r@`Kr@@sr@} Hr@ѕl@r@.r@Sjr@C2Rr@[0$r@ (r@sy\r@:r@N/r@xeOt@ Wvt@Lt@MSџt@t@^Ert@i3.ft@G߷4zt@q/r@n~hK"s@L#S+s@W5t@uM t@&,*t@~)t@Y>yt@CVt@X8.t@pPUt@!?u@ةq.7 @#p6]@M #@IxUzQ@0O7@(Le?t^Fcv@&fv@H04v@K2v@қ-|3v@wE(u@xP֋u@0}xu@ Mp2B@9zŚC@l uD@|/G@7H@'%N8]E@pCU`G@" J@t0K@IQɩF@aWIG@oYG@<Ě%`I@c%xtH@H@ f|I@ ; 3@)d>@;t&{;@SxbhlB@@@ ZE@-B!E@"'&t@w|. u@@*W#@ @$ą,@7Ef4@{؝b%@5[M v@e5ov@{5d`u@ﷇܱs@n-f̟s@s@ \y$Ds@C s@EZr@H7s@w;$t@P~zt@(륗fs@_k܈fr@ >r@dq@(n6fr@fr@S:ƃr@ĶiPr@|r@v:ot@kp:s@{t@XP)^t@EUU;t@r7t@Ks*t@#~ja;#t@,Kt@ta&t@Azt@Gt@6y\榒t@0Ut@Z{Ct@ϽKt@4tt@O^s@Yn,s@!+ Zs@^_BXs@ush9s@K ȹt@-(u@iۇt@_.u@V%$u@ϲ8%u@yBZu@1Fu@ط%u@{]*u@e˳u@t;2mu@Ef9ѫu@Է@46;@wzJs@o*@gQq)?u;h῁v_v@v:]_v@in{@=%?Mu@!ΐu@{Jv@$S.v@%ӎSu@=Cv@)Kev@]qK@CDP@/C}P@PvJN@@;L@+4k N@sm#M@P@̷]vO@)gP@zP@e:3QzO@g|M@%nH3@QƮO<@辜ٮE@4>݊H@ HidK@ob~nH@M~0#J@HZRFt@;Ht@/EXMt@GWAu@Mk-t@; Y]/@ǛT[/@)?<&u@|@u@9 xr@|Ir@m)r@Lf+q@Wr@-lr@매^s@H]Qs@ebs@6q@9<^ p@=p@2s@e;Ms@7OQєQs@i*r@Ks@_s@GW[r@jW.Wt@t@덑#4s@tEs@Om6t@Y[^t@A{UDt@_ Gt@,t@PBt@@htt@H>.)t@mgt@ t@X*̟Ses@/wFs@Wats@54s@.Cs@Z|`s@lQ u@s/u@YẆ$u@?XHvu@nu@EMQu@GDeu@{du@HbFu@ίu@ru@a9N׷u@ ` @mSk]?UQ-Y@R Lv@nߣz޿](`?UvHoP)bv@aW4u@>̐u@2v@5 v@!v@ټrv@וR@+N@{zQ@;AհN@+OBQ@ZㄯR@SR@[|+nq@ `G$q@N4qgr@Ur@W r@r@gH9|s@nШ Xq@rYrp@Ɓ3Ԅjq@&Cp@)VR=tq@4&q@*q@qGp@YqMp@\=Es@DRyFks@dA(s@ l^~Vs@?9 "s@Zyl;s@"s@ r5rr@L1簏t@m&X}t@WՁBet@aFg!qt@.W?6t@?Rs@\SP t@gb(+t@*4jt@N @t@x$t@`!seЦt@~{t@/xPt@W_'t@PКs@PXs@( s@4`ls@As@cs@lu9xs@ͬnis@Y[s@Dywt@C}u@Yt@e{Du@D,u*u@$yu@e۠-^Iu@e0Nu@lG'qu@>*}u@e쬧u@UqKu@Tc"[u.=ev@h<^ v@ .u@ou@ 7v@U\/`ܟT@ .T@D5%"/R@tZT@7&"U@ #b*S@EBjS@0^2S@7߿5R@@EfQ@10<0@KvH@ ^L{K@3|YT@uR@󈉅W@K!ms@c$t@n#xVg:7&C *q?wq@z#jq@$r@Yˊq@߼%q@b6Kas@XZ3)p@eP*5p@䌅Eo@5hQ2o@;&vp@}T{p@q/o@|M> q@[NS[p@@eip@p@ep@Y8_p@Lo@up@vf'(p@+3o@eݨp@k$Nop@@?p@hVzQp@P/fp@}Rs@DV}s@㲉K6s@EPYqN"s@d'5 s@ bNbKs@`t@8qt@$ t@Ďl(t@2.t@gMwt@"UYt@7wsHt@sPt@yN6t@0t@t9t@St@?Ct@') t@;n/Ns@u*~s@czs@Fns@4bfs@_ٯs@wA ss@iRWs@9ZCb}s@³Du@!jX:u@qu@b?u@dYu@]FV#tu@at, fu@={.Su@)zu@]6u@"u@19u@v@989v@vPv@ETBu@[̘K!v@Š@bQW@xc~hU@zSCW@.BU@wVAV@ bY@2٧T@42T@YՐS@F~!U@/JkV@OA@{,@ Y8I@xdY@x5R@^@=F\@#[Y@1|p@Ϋo@0q@_/po@w89o@Ua,(o@:,o@|no@ So@p@|̀?p@qo@+#F*o@K^Sn@-Pn@w!n@}& o@[ŏn@Fo@z6o@Ht o@|bo@@hϷo@SWZo@18do@8Vp@tg+p@HE1p@aܛ p@\YkA+s@Ӣys@+ls@/s@-bs@ƀs@XP u@BVt@^t@ X@ڷW@).FvZ@ͺXvX@HvV@ەU@eWGk@f5[@^@0ڰu@D5Tu@"Mu@du@*u@;-.Du@[r@1er"q@fLr@Tr@t듫r@^r@saI^Vr@#q r@{,r@C;Br@10q@1|r@ 1tr@`̖Ir@1)lb)r@vq@Qq@hN>Br@9ŧq@Gpq@ݻ%q@9Ip@̔Dq@7eKp@Xp@:p@%hYp@pc@,Mc@zb@1fc@-Q}b@63/b@q{ua@(Aa@( /a@MT`@W* _@Ep^@Hڦ]@($ `@K32r\`@\_@]6e@!~},|e@ȴmOgd@(;c@wth@H_li@usj@{6i@G3[i@hyBSk@U;j@0Wk@ÖUOk@u k@r}iGl@$7 k@K>%l@ P~l@\3Tl@( bg@ W,nh@D]aeh@Vp1g@]%f@6Fe@zztd@%l_0j@Ёk@Ik@]j@Sthk@ԡk@1=*ǭk@|Wl@ X.l@tPȻk@p2m@Qs@[R0s@VFbs@?us@MݛCas@R8u@.OUot@᪎;u@ks@`s@rs@mVs@ G; t@t@s( t@uPw_t@78lրt@u u@vu@mu@B¯u@C]mu@4@ճu@ԓgLu@39#u@g yu@7u v@擧v@|A (v@x Fs@pir@Cr@؛Mr@qABq@-eq@pk8r@ճ{-r@2r@!fr@ir@=̕zr@Gr@7l[:r@0r@vaYr@t֔QTr@ q@/kYq@\Vq@PAq@ 2]㎣q@o}q@Y4q@v q@9/q@()Kq@k5Mq@. hq@]1i@xĥg@&c@f8TBb@\Gc@t5c@ ad@y/q0b@aܲ g@Kmf@QG5Z|le@@Zyi@E=j@"Mi@Wy]h@ki@8[yKh@ӤVk@܅x/k@FJ j@&,;j@uzqj@eBZpj@0rk@s. l@A^}s@11s@w/8^s@~6 s@K}}"u@AFHu@7Bq u@\HVt@ D|"t@{s@|=j"t@@.$s@9s@aFt@#0hr@WNr?r@ܢq@iJGr@_r@s7r@]r@Wr@;:r@$YEq@[?q@hq@5^Ilr@ q@=q@q@? q@w@q@ ќgq@E[qM]g@0ю0g@K8b%d@Td@;Wc@;Le@he@J54Re@ٙ/5d@0g@"|g@SMh@ci@ɷg@՞ai@q$j@At@|Gs@uŵBs@z1Ziu@>ӦEsu@@5+t@Ov67t@btkt@bct@k"t@<8bt@t@7ܓ,t@[珖u@'Pu@өJhu@"~u@dq u@\h u@Ur_s@1ƚs@fuos@7$}t@TSNw]s@CÑlPs@'s@5]r@MUs@;fs@p}Zr@)2r@}٨r@zJ$Ur@rVwr@ aZr@s"yfr@S1 ]q@`q@_f@yFv3g@z#ryYf@dDaf@$]3f@a8jh@g@l th@jX[vh@? i@Ȫm5t@և,ɂt@m7eHu@rNu@/CE Bu@~u@-8u@ht@wCQqt@gcu@i`ru@yu@,zLI{|u@P>(-t@ lMr]t@Cdt@dMD"t@|z:s@Z!+s@ݱ&5r@!s@+s@q^Rkp8s@nIr@'0/RZr@#,r@&r@$Dte@ Cf@(,g@0LoH,f@s哕h@Yk=g@;Vg@cL$g@s^t@Llt@/fKvt@Wezt@_ShEt@u@ B/u@ q[t@yu@Nu@+[xu@$`Ru@u+su@8vt@WGk t@g@g1t@ ߞs@ s@`ߐs@ⓩ]r@@s@o7ws@kVl\s@ws*s@{#s@a 9s@ 7:s@29~r@f|r@оS g@3\{k@Ӿ^(l@'af@`\g@*,h@E#Dg@Mj@¡j@O]Ft@Qt@t@est@9t@yԝt@E : u@k&Xu@؛OTfu@'Yt@C^c'u@܈}2u@|[N8u@]RNu@[1޸u@C u@]]u@-p:u@#u@ 0|at@嫉dk@ody2~k@q2Xk@8)m@hw%xf@DwLa g@Pg@f@c@R Cd@j]Nl@! l@/@j@CfӶj@pej@RR:Kl@=G,|l@#zZk@ιk@22k@KbiGj@s55k@බl@zKm@ nGv#q@+<$!g@^Kf@z"f@`e.h@eagg@Tg@kHj@idh@h@#wY!i@HdQi@ܬi@˰cJi@1˞%u@:ju@eVu@hU-u@g#VLu@QIot@I=9t@3t@0,XIt@U[xt@q/Ut@N t@-kУt@q=Gu@p+u@Ou@s#*gu@E^үu@__kc@՟|\2c@Ȳrb@uNc@Q,\8b@Gِb@>e@wm_?d@ʚd@š8e@1,Ne@Y`k;>d@AO:H@d@g c@ c@uNc@ c@{ˋl@ 71\l@Aȓi@v Li@1j@ ]\j@  j@3!l@0(k@auRk@G%sk@AYl@{ftm@²l@}Yl@9@0n@Isdsm@_6<ϱm@Yrwn@n@o o@Y@.o@n@ Uo@ *~%o@ ,8 p@yI1o@-rb"p@P鄴p@``Rp@T$L>#p@7^q@=q@Qwܔq@M/p@,:p@&펭p@Y"=2íp@5q@: nmp@;:@:q@ Ap@m+Jp@u5ip@m:=p@.p@K|ʄCf@Ye@|$Ng@Xf@L9_}f@rxf@g@9+&h@;qh@eMg@eYh@3d2i@Su@%;u@v@7nd,Tv@PRR$v@Vt@t@-?t@ ect@{uDt^et@TIqt@?Woht@jXt@xM u@@d'u@X>u@Fu@Xv@Y5u@ &u@NYu@ 7u@äu@7.-4b@q"c@ӭ^b@+ e@a3e@iZd@tVd@aUd@ c@AV\c@xNc@sc@;ˇl@yʆi@kj@뢠Jj@\FE k@ (Hk@s j@.IAk@Rq@aq@(%q@ȇwq@;l@uWl@w l@)DVӉl@PjԨtn@Om@U7p"m@^m@{5I_m@T5m@-cy n@m@EAkn@_*n@uvzn@eZn@n@UMoKo@14I o@~)7o@ #boo@59p@ p@Y.sBp@O\P[p@eq@Ggq@$V5Aq@4ڰp@q"p@p@džp@Ap@{q@@Cp@mZMjdp@` 3vp@8Lz1f@p e@kcG>6g@|lh@jPih@a)JRh@]?I]g@0/ʫjg@3]Ai@fAi@X&DA| '7v@ʸH玐sIv@tpZ9v@7r@PQicnr@ -r@=m4u@D@v@#Ku@Gڴt@Ot@ϯ.t@\t@9Hut@+Bt@V4?t@Qu@ⴖu@eu@ ,Igu@u@Dw!Ru@ u@S#r@`+Hs@7Jb@Jjb@1xc@_?(b@b@. e@m(Ld@*% e@=nc@ѕ, Kd@dNc@coq@Ӄq@ݦq@59Fq@Wnq@xyq@|>tr@~r@er@mLr@eՂOr@nr@pq@9~+r@y IIr@8r@*&pr@7R>l@ kWml@ߨp_k@)9kl@bUm@KVm@h&<Ŏn@6ql@[L`m@+9Yn@ɓkfLn@l n@Z o@Đ Oo@bn@uւCo@EΟso@L%p@xq@sGq@ӳ3_p@wȺ`p@'3`p@fZp@D p@L9p@-p@n4p@u~wp@;'o@ѣ+6p@Aҧ7p@C,n3p@>swf@Cx9f@5D[ze@M.LNg@7'gf@Zh@kR,h@hXTi@v@}\BZs~Ov@ԏ)|^ԓ@0>?շK @L5E'@Tl$[?qwar@y ͦr@ r@Mr@Kq7r@\]: s@9+r@ IS*r@[ˬr@#0hr@57r@ev@%$$v@ Kv@yt@ ot@bt@'t@Ys@(t@+o+t@ S]t@xt@avDrq@l@,)yk@_k@&l@ _l@o.Y^m@xٜpm@Pum@)zE m@`X†m@ _rn@ Vn@]n7o@$p@솤Mp@p@4p@|Ƈp@H Ao@`+o@+.p@1=o@po@d@p@{GYf@ُe@7We@%}~<؄f@MT%g@a/&h@Cw67i@ ?xw@B@"&@7m~{V@pϼ峵@,nɗ=F@ӥ@ hDz@YȀ'!@a<9r@~jx r@3r@*-mr@xMǛr@,Zs@,[ұJs@#"Wr@r/ s@ uXv]s@lְLr@W s@OdXBs@I;s@%Fs`u@äw"O0t@Fرt@h[(u@\t@ 43LOu@%t@Äqt@ t@dV<:t@턨u@ JPBu@mK!u@%#'t@ϕs@a@]t@B t@\Hs@!<8Bzs@MZWs@Ys@p&oɭs@̹o t@YXa@X2a@fVe@p*jjd@PFc@_;a@qc@qPc@%vmo7c@1Cb@}dΎob@c@S<.9j@c^k@ Zi@9j@DlIq@|N ;q@,q@^3q@򳣑p@QpHq@/q@s~Mq@Hq@ 9%q@u\q@y)q@$pq@[\Wq@Rwsp@G8q@a=p@GSxq@A سCq@Mbq@;lpp@s$p@8,{*q@,?-q@uDVp@De.DZq@yasq@dB(q@sp@3nNp@0>gp@oOp@'*[p@Aϯp@`Ll@uXk@ U[k@XNyk6k@ m m@W m@ Y8m@TaF %n@pLn@ۋֿn@ ˃Xn@y"jn@9AUn@9Qa]Wp@f NQp@Ilo@ݾoo@ދoe@e%ӎ e@yu#|e@%6Vg@u^@f@VCbf@Eٛh@h6cYh@1-?pF23@X% @L $@6 @'s@OAk@s@ۯ2s@&^es@[s@>='"s@p|s@>/s@uW1Bs@]QAs@䥿\s@4js@av@<Φu@Wp!)v@N%Lv@{EؿY?uPe u@Qt@JLsQu@Pnmt@/t@GKWt@?SFu@?gu@.Rt@4Vlt@t@W&s@+ ss@c>Wҟs@s@F.ws@;ig`@_8CMa@[-4a@u&a@%o\#`@ c@fd@TCcnd@q@yCp@T=Op@1&hwp@FQp@#Ϛ$p@sY_w"l@3a l@yl@SDrl@@Rm@k7yIl@Qm@WeJm@Yn@N0m@ೈ;1o@|]4p@8o@:Jn@XIoo@+yf@s[Bg@$~ g@Lxg@o$[h@=Fh@!mh@8k_ i@`@UVR.@"[4 @{؝@ř @7 ZE"@@pKms@s@4 ks@_٫s@ ^as@K|7Fs@>s@ s@_s@4vs@vT7s@ Jr@M|es@iRr@bs@O9Qs@qjR2t@s@!O'@+aX8v@au@S: ?;Іt@L1>s@_t@%ƒt@-Rt@U" @Og8u@ń=,ot Ru@mV7u@uջOt@8nu@t@e%t@mby<.u@> ۙt@:Ot@IyVa@&a@+ҕ`@1U0Fa@' `@؏'\"a_@5$51`@sH]_@-"c@Db@ݘsc@b@.#b@7Ǧa@<@b@K_a@b@mxťc@ Ue@Uj@Ec@0Qd@Y :c@ c@Ixd@p,Ve@aCJ8on@K+Fj@wi@TGjj@Ualj@3<i@Fh@[Ilo@/uźn@p<)o@H"m@{OwAn@"o@pmp@1s]ԉo@ o@uQb;p@%+A%Gp@Lp@ j p@K?p@EXg5o@V?pp@ۊX!p@Ȭ(oQo@*#p@Y1rKo@A"'o@u%n@cv`Jk@Hs-#>j@t}Cj@ ak@=l@mmm@dzl@ɢUl@eY.nwm@|3V|n@H4ah@~ ji@KTfi@ ydh@.&uag@ ]h@@+0dh@+te@Y~f@ f@yn/g@4:hg@`ֱ7sg@ %S{f@f@#8gh@oh@x,h@Hޔyf@)sg@Ad#5f@8ee@ Ie@6Je@U |4e@use@\),e@aJO@0^"N@GN+@#mG%@!Dt@Da*pAt@Cut@W,Jqt@U)^s@-'[`$s@5=s@8֡v6t@Яs@we^s@Cڙt@s@Zkt@ZKVWXt@}8Hs@c @ݨ%@!Y.$@(0@̢ @7?_ u@WMKv@}+y#v@-o&u@uu@k@ޓu@C&[Cu@KZv@䟗u@mu@f (t@Vz\@dNsN]@l\@QKCa@Ylڎ`@ Ba@dj`@W@3 M^@ˏw^@Ո^@**UI_@ f_@)ߴaU`@#48a@ՙa@iL-c@S$0b@_Qb@Lj Jb@uvqc@}_c@A Ab@D*&d@ L=c@ N d@d@oMd@\$Ve@I}e@ϑm@q|5l@ifKk@)xG_m@_RkPj!m@Lvj@hn@%n@Ccro@8{0{o@n$m@31n@O n@@~m@LB_o@xδo@xio@'i" o@ #xo@ qb-o@<>L=n@Bn@ĘOn@ki|n@L 찁/o@t4!n@˜n@UP>o@4n{m@Y-n@m@' dk@\b5 j@1=k@#]m@Yl@(h@qDi@xg@T2g@yi@m1@h@!|i@GDdj@Kqh@F+^h@2rh@cTBh@eڬg@Vn…g@7]``Gh@h@Bg@ֹ<:g@2;g@ g@MYzf@1Ei=g@81[lg@o,Ef@mN)g@zGcg@gݾf@48f@XW`#$ g@eTpf@;f@;὚f@RBf@ن we@ܹ0Ke@v>e@*Fe@e@e@Bd@#)d@-ܨd@Q+d@9+J@\M@KvH I}O@]](s M@q:C@c9S2J/@pᖝ*@".@j=u@A ft@;uCu@Rgzs@{vs@[[s@׋œas@?Is@}SHqt@Kt@pt@ DWt@w aHs@s@J=Bs@Y'xTFs@ CQs@Dy(@e |(@Mz@w@{?j{s@@&u@qq/u@9{8)5u@?Fv@4Cu@ֽ$v@ \@kəޜ]@  ,\@/^@{"a`@7ڨ`@θ`@ #n_@]778^@9rIz_@~a@[-y9b@ xb@71a@pa@Oa@Sv:+b@t]nb@]778#b@]e@[ie@"7e@kR&c@|5Qբc@%+Pc@hV[VAe@ -h0d@c@B=9d@md@ɎVd@e"sqg@7_g@%k@#^l@t%h@یN{i@v@hi@~>Kg@e[fh@ T+h@Wyh@Qh@8D1j@<:k@ϳ;k@8/i@mj@H`i@Dj@D]j@*X"k@_i@Lh@t)Bi@eOh@ǁg@9h@!4g@RJg@=SFg@2ayf@sg@n e@Vh:f@Lmf@ g@`f@ ]f@wQf@x:f@T)}f@w@~g@?֗g@af@[eg@&nif@o>of@l'ܽ.f@ڏ ~f@I^yf@,VUNe@YYce@9MR&f@MX7he@5:15'e@Ve@de@uqe0e@$}d@֗ d@ģCG@XU[H@X! J@'hk=M@fN@q^L@MM.=}N@4Q@m`!NP@ovP@{h+xAP@ IRP@g^5@Pu@&axu@ vöNu@XWu@ru@0@Bt@DUp%u@;#5/t@wat@-Eʿs@Bbq;t@Ot@#vt@7peDt@Q͈n t@q^zs@sl-@ֳh#@QȻt@#%@h' @z@u@ ${u@}kq\v@c">5v@]oEu@pմu@}T^f3v@Di޶u@M nu@AVʅu@勬&0u@кFat@!U?Ku@[t@)Έ at@˹ۨKt@%* |t@3mt@a @ܡ(+@rv@xeA-?Y 5 @hQ>@ó_'@hg@J`W @ŵ5} @Gn|^@E1$ _@Cj]_@)d `@5jUJ`@Ch2a@e@~`@F5ra@w`@+e@,}Cd@đ*d@e|{9d@Knerd@g0jc@P_3d@ud'd@?c@#ؐc@ԗnc@c@$ABb@5h@cg@@!KjR@@l# :@'xV;@|Ԗq5@XM.@Вyu@.v@K0s.A n濬z{@\@pk{ v@GS=u@{P:v@\/ru@ u@Jdiu@a75t@g)u@-/@;pϫ'@]r @%X^t @<"8?gDj_@ǔS`@3c_@><l`@1 e@ 1pd@/{d@Nc@5,c@΀Vc@L} c@Yǧc@@`SUb@-4c@s[g@E;?g@{>k@Nj@wj@Gi@R7xk@a*ٍyk@Ŗ@i@KGQ3i@$JּDi@&h@S$j@L}k@v k@xj@8¹=j@?@j@e0j@/8fh@àh@;Jʂ.h@売Nwh@AXi@.Ɖj@=Nf@>dGf@#d nh@@? >i@czh@Gg@jؿf@ f@8@Սf@@g@Djh@RGg@>1d@l*{!Pf@Rf@ պ^e@Qs:Je@bo8e@.+e@W4e@+ wpe@#ۋ6ke@ŏh9e@le@L*f@=Mlyg@H=Ó g@^Rf@)fe@\+e@ Djf@6f@_~f@3:e@b*e@8He@^d@h{!.I@`,+oI@ Q%oJ@I'L@hON@!"P@?P@O_*,Q@EOQ@~K?R@ }(BK@-bͤI@tAF 0K@(ML@K@а: M@J˜J@+7>K@_Q$rI@ES@\ӏS@:C"S@_m3W@C#ףS@cU@{V@SܞU@^8ƄU@T@%FNV@yͫ:V@?uT5@O5B@{A@;WA@=qo@@GaD@k'?@ѯa'B@4y<@-:@<@Qة?@f!3@\e=2@][D9@*)j 8kPJ"@暸 @#+@긙(@+'v x ?qJ[,/fbvu@UB7@&e.)@5c!@+OD(@y (J(@&<&0@:5T!4@ļ4@ێ14`@k_@fS_@8hA2`@˞d@TCd@ʿܡ-d@!Qc@~42c@\c@c.Wc@(b,fc@ߠbAb@n'b@*cg@ wh@*|Gj@U&)j@],j@9&Wi@#i@Gj@Ymj@IJCRj@DClj@Ml)@i@8,Dsi@7si@ Ci@Mi@_Wih@{ιwh@P\g@HrZSf@5HHg@h@yh@%h.g@_NLg@rC/lg@Nd@5d@$f@=&9f@ld@,bjd@|2e@ Hd@盵\>Je@Uڬe@Pklwe@0~e@f@}RVg@ VIf@f@%ݡilf@pm!tf@Qe@e@g=e@WZ@.ԫ:@} wC@fM,_D@D@A@@p\B@B@qFF@@W_9A@^r>@SA@-7@1 ڈ.@W95kv3@V/QO2@N:+k?+Bt#@|@@Wɣit,@e{u7$@a4+ۇ;@ # 1@@ 1@8@(ϒ 8@w QD:@9iI &[@AtQF\@+X[@*\@_@d8_@"Qv^@Qb`@ŚR`@[Z f6d@ c@כVTc@[Tc@lc@.ayc@ >g@]Ni@yY -i@=Di@kkh@\2i@5kqj@Lth@Q i@>Qi@qj2kh@]Sdh@{.jh@H h@¡g@HIh@ g@ Gae@3c<8Lf@ ;+9h@]~ph@Zh@4cNf@"Lf@a:vGf@7g@yL h@ag@/^f@p:[d@3zd@ͽc@c@p-d@LAm f@)e@ |e@N}d@Y=ׯd@;!Kd@Bk(Ce@ۼ̐f@~@Qf@HD@t "D@{pw>@@8ƖL<@dcA@?BC@l#f4@\&yd":@ϊ04@=D\6@Cn6@'m/@&1@Z@o2>@<n̡Z@憂uYZ@};[@jSZ@Dj\@Y΍]@싨D^@^@{D+_@[Oc@ Kdxc@J3'c@f@#h$g@|Yg@6}ph@Xhh@hJi@0^&:h@$zh@](i@g@-cg.g@-f@O;3xhg@= Ig@(e@כ f@3jK&e@=Brw8f@V{evf@<f@0yf@K&h@g@YͫÛf@Pg@(Ⱦe@7|ae@14se@sDd@# d@GO9ۅc@llex"c@ c@-f@l$e@H4aue@耤e@϶d@SLd@襆d@D/d@nX@Ke@1+ae@XFf@%i&f@ee@8oR3e@1}e@:_e@a?me@n:e@}be@֩MJ@70I@w^OK@Ԡ2GI@RH@ uT@!jGW@/(W@2LX@)=ʛW@m1}X@@SEY@6lT@uSJ[V@d\V@,uY@AX@r@\$Y@usHY@>yGZ@UT H@XhH@YyD@z+\G@!E@\F@ΖF@^N@@D)IC@o:۶0@@ E@B5G@qs D@D<@HIFT8@8;{9@:@#cY?@ãoB@~=@4[@ip[@S!W[@DEANZ@5 S\@~[@'?ʑ]@تn]@7v]@Uc@!%`c@6e@]Td@Yxd@ac@kcjc@ƒTc@)7 )f@7e@@;Ce@oe@Ӑ]ge@_4e@2bnd@'ƣd@GAd@][Nd@@d@d@w#d@Cr*e@âe@Yf@wk e@abzTe@VDd@`5e@Fce@4d@ve@a}^Qe@Ome@-be@ȮJ@Pf;^TN@$H K@ͲP@hMOpP@GmM@j `S@ݘiR@PNR@;S=bP@_iR@3^O@{4U@yX@t`@p N@@m=@_A@oķeD@dC@wA@蘘ӳC@#5\@*T[@sk[@9o[@CbZ@Icw\@}/ҍ\@3Lb@nDf@h e@Pf@f@}{kf@%Lf@-s54f@Ue@8)Mːlf@eH"f@Ze@%.]e@ͧ3de@i} d@3e@c<]d@Y) d@Ρc@Ur_mc@ MI $'c@: Sbc@Iic@T6X@azCbY@ܽSY@U_~Y@Є91X@Y}Q^)O@!M@~\vZ@lT;[@H?!W@5sIX@ІOZ@1Z@,l)Y@7W@C a"Y@KOkP@yeO@C%P@y%N@'P@ᗽO@OXM@(YQ@^NoP@:N@ %P@_gP@QM@sAY|N@UsyO@P`K3J@?H@[6tI@lG@17{[@eR[@ 9\@ \@M`@6a@Da@WCtb@ Ob@OpVie@Qpׅae@A`Тe@ e@A B.e@e!e@L e@k{.e@Eb.;>e@$_d@!hAd@@d@99n*e@_ d@J}e@#Apce@Ate@_c@Vd@ntId@Xdc@v{c@U&c@)eb@\MCd@Gb8 e@.NXc@[d@3:d@ d@<\ivd@ٜ od@| d@1?Gd@ Sc@fcvc@H)2c@@c@d@gЁ0d@SbBLd@N+zd@7bUd@#?d@3s5d@'y$`d@[Fd@SDoZ@Ah[@q" [@:-@cQ@rR@XS@K4$:%R@YtR@xax̂S@'=]UT@$7UdU@^eV@~_U@Ս V@;VZ@-QY@Y+xX@BY@/XQ@'WQ@t%տQ@%!ZR@`ڸFL3S@@X]Q@{wsQ@yYP@{֚P@D-pQ@8j{O@D oN@_WrM@MF.O@\  tL@@xI@e\@|kM`d^@27]@(yu[@m/|W[@ uX [@IQ ^@YR6 `@/5`@ó;[`@,a@NFa@ϑta@q#R#:a@Q`a@m^Ž,b@2b@ b@%d@L,e@CGd@ͥve@-ud@@s1aOd@Ś@d@=Xd@p *d@jd@Qtd@Q8-d@c@ݲ2c@l$b@o c@Ac@+.jYd@UGd@[jc@P//Tc@b.;Vc@3 _wc@+ic@-Id@ ftc@1|!c@uf$d@pOd@qԕd@XYd@dC4c@*qovc@񯊜c@|c@v>bc@x~6c@K-c@J)c@A&c@B:c@Wc@rc@}mky8Z@^g7MS@!`,A)T@Y-U@0gMU@W V@Q\1V@1.W@QuY@NFY@ۈ?$Z@XfX@OlOeW@h3S@e4R@T@H3T@S؋P@a}}Q@piFЬR@mͱefR@s<>P@P@YtP@hs `8Q@Ma[@&\@!>f]@pٰ]@ӝ^`\@k^@u&[@s3^|[@Z9tZ@?_@*d/r_@㌼_@s`@2|1`@͜vy;`@yɑ`"8+d@.hk2d@dTd@eu, d@@nc@'vc@L!Fc@{rq.d@̖u}d@llac@Zc@=xdc@U `Tc@~!ac@Xʢb@n b@{jՇb@yb@qݒc@p!1d@kԞc@|,!ic@.(`-c@tsc@kq'@'c@Ll [c@ئ/c@A:Qtc@,u[c@x'>c@%c@3h`c@h}c@铸j)c@S?c@UhWX'Z@ד|Z@'ZYT@jG2U@LV@ㇸX@Xp9W@g`Y@1#·X@B Y@ 7T@Q AT@x MV@\])V@R@a})S@>S@aoR@(pj*Q@,Q@_&R(S@-R@O S@)_S@)ʁrS@ͷ+tQ@s;^\@;]@F\@#@| ^@p3]C]@SG ,]@+z(R^@tj^@@J]@[I[@&䧕\@7T\@/\@UoT|_@C( `@Aо_`@Q`@]0`@ۨr`@/Ha@%3Oa@rՐa@a@_wka@ɘq va@x#a b@$Mrvaa@53d@ D,d@4`c@wjhc@Ec@d,c@_v#,c@4ݹb@+ruCb@eLDb@c@ Y@e8\OW@,kW@e+Y@JX@ŮY@W@XV@YѱU@dpp6V@;JMW@SyV@ΧT@^8?:U@?jwS@9 S@v!T@eOT@LgRY@eX@;X6 :X@wX@=X@VW@-^@ 5}^@3pJ^@[^@]|<[@h\@A7[@_]]@-]@ӸI\@4G A_@|T `@-Vi_@L3`@ȕ`@!ϓ`@沚wea@`68sa@/*a@T&a@Xܒ/{0c@;b@|6b@֭0)b@<5Άpb@&Kb@MvLb@e$b@G&)!b@F)EIb@Ž:Kb@]>pb@peb@8jb@%d_b@¨b@PCcb@eb@hhb@1Yb@w!0iZ@MIt[@{w.Z@3D_'Z@,Z@~Y@.Y@ Y@M\허Z@= X@esqW@ Յ2&W@5~X~rV@0VW@9DzU@]a V@D"ZU@i!U@?z l|T@t =oU@SY@Q(_@p^@d+0[@\@V!xl[@;ᓼpZ@)\@EV \@2tGڤZ@aq1RZ@ϟ`[@y ;2X@@9Y@d\Y@7wvX@t`W@lXFZ@{*Y@M_@DY]@J]@$m+>`@:۶_@bi2_@uSkeq`@=L7`@ˇ,a@J<|u1`@Sr3Ha@3(va@5a@pn#`@p5j=1a@鳨x;a@6i`@qfa@fWa@T\P>a@|.wa@-8qa@ Jʲa@5a@La@:Na@K R\@1\@S$H]@@P\@K}Fi_\@[@.l[@ԓ [@[@Dn= ^@v)W^@>]@-]@(X7`@ ~`@_7^@+`@9}=_]`@Ӷ=`@2g5`@ma@An`@i9`@4`@W`@$ܬ`@;wUa@D ra@Ay"!a@ 7pa@ս_dMa@K~ka@_c`Da@-]@3'H\@ "&;\@/:}a@\@@YyJOZ@5OP\@XaDY[@䳆[@/aZ@<ϷZ@4![@[@6_@{^@ݽ`7^@U&_@U+T_@0`@0=`@] `@fP;`@l ]_@(`@{ }`@v?o`@KͰѶ`@xX)`@r`@r>=`@&&a@*Qa@vHa@^n5a@4 a@,oIa@_]@Ś~]@G"]@W$]@4 ']@4z\@+69\@B\@.5d[@'WKK[@T[@q[@-3!Z@ [@iԍԲ^@/yMYv^@^]@[_@&uY_@BԔ_@*`@G|kf`@u[`@hkW`@:<_`@,4{r`@c{.˝`@YNZi`@`@"#`@5 ]@U$m]@;e]@{M\@)I\@J\@=ԃ]@]E^@#+_@q,}^@ߠ!^@lZr%^@mɵ_@;B*8_@e1A`@ui*`@($u"_@\x7`@E+~?I`@ȁF*_`@Fy`@`@4T C]@Wx^@Iu]@3x*r]@ g]@Gj=]@ 0^@d^@W B^@)N6f_@\&i_@0tU0_@ϫ^_@iO1`@6:e_@˨_@l,^cl^@%k]@,^@cF\^@oH _@ڕL_@;齌`@-^@ϔU^@ۉI__@ DATA-END ATLAS-IO-END eckit-2.0.7/tests/geo/eckit_geo_cache/grid/fesom/pi-73bfeff62fa8f78f078a940408587118.ek0000664000175000017500000042405015161702250027237 0ustar alastairalastairATLAS-IO Tg .((yaml0none:0 METADATA-BEGIN { "version" : { "type" : "scalar", "datatype" : "uint64", "value" : 0, "base64" : "AAAAAAAAAAA=" }, "elem2d_shape" : { "type" : "array", "shape" : [2], "datatype" : "uint64", "value" : [5839,3], "data" : { "section" : 1 } }, "elem2d" : { "type" : "array", "shape" : [17517], "datatype" : "uint64", "data" : { "section" : 2 } } } METADATA-END INDEX-BEGIN Pnone:`#none: INDEX-END DATA-BEGIN  DATA-END DATA-BEGIN              (  (**+( --, .&%003.27687 :  ;6/;.?!B "C!"DC#=I#$$&&'%&O''K%()(P))&+*R+*+P(,TR,*.4/5;/UV/.0X301X0%10%2W423Y3XY324U.4W[425/V5\]6_867`979789:9b:9;_6;5^<ED=GI=ge=>g>#>@i>=#?Ah?@@>@?jA A?BClBA B!CCmlD"<DEmE<dEdnEomFGeGtIG=eGFHHpuItvJIvJ#IJ$#K1%KL1LX1LxZMK'MyzMwKNONM'ON'O&)PQP+SPQ)QQO)RTR*,S+RST,-U/.U4[V\WYW2YXYXLZZX[U[W[\\5V]\^c^5]_c_a8_;c`78a`8a_a`b`b9`cc;^defFfefqFfrqggeg>ihAkhjigi@jjj?hkABlkBlkmCDoEnomopHFpsuqpFrfrqspssutHutvtGHwxwLKwMzx{xLwyzy~yMNzw{|{}~yN~N~OQPSSRVU[V\Y]]]^]acndnrrfihkijkllmoonpqqrsustu~~zz|}|        !!"#"$$#'('')*+*+,+,-,*/102/36 54 4 8       8 <==?@?BACDCECGHHBK&LKN$P$RQ  !S !!"S"TS"!#%K%V&&(&%(X'(WY)Z^)XZ).)'**_/+`_+_*,`+,-a--ba.-..)^//_c0201110d2ed233cf3/c45h4h6457g5 76467ig8 :9<9 9 898j:  :j8;9j<;m<9;=p?=>=><>oq?@?r@AsFAFCBBGBDtDvtDEvDEDBFxyGt{G|HGBtHPH}zIJ~JKJ%KJI%KLKLLNLM]MI~NOOQPHzPU$QQOR RSR SRT#UT"#TSUU#$V[V%IWYW&VW(&X(YXZX)'Y[IM[VI\[\\[M\]]]\M^b._c`_``a`,aab-.bab^cd02e23edfe3gih5gjl;lmlnljm;lnlnoqomo><o<mpr?p=>p>qrpsAkssxFt{uCFuECuwEvtvwwvEwuxsxyyuFzPz}z{||G{|}H|}|~M~JLN~NONOOQUPQRTTUzWVWXY[\]]_bfcefffiighgnskkknmnoqpq{uyvwxy}{~       !"#%&'()'*),+,-,-/.121434576786;2:2  =      > >@? @? @  ?BB<  FIHIFHJ MLNQQ$SQOTTUS  LV !! V!VW!W"!""X#"##$R[$$R%$%&'_('('(*^)**a^+*++ba+,b,-c-.d-..0./ef/0/&]/&0/f0d.131121g2h:353ig4k54687476l9;9po:;;p9<D<ts=> = 9=u>=ou?A@vw@w?AwxABACyDCDCBCxyF|HFEFE{F{}G{EG~{H|HIIKIJJKKKIKJL LVMNMNMOTQRR[SQSUSUTUVXWX"WYYZYPZPOZYP[\%$\$[]^_)^a_')`a*+bacbcb,c-ddfdcee/]fd0g13h2ghq:hiji35ji5jjk47kj5l68l8nm7lmk7mknlouo=9opp;rqhq;:r;qrrqstttDzt<Duv>vwvv@>wxwA?xBAxCByyzDz{}|||F}~Gs~sLMLMPOPTYPSRRQUTWVX[\[\e]a_^^`becdffehggiqimjkmluoprrtzvuxyxzy{~}|}~~    "!"#&%)(*'*-..1(4546766 89: ;   : ; 9    <= =    ?> @@@AB@CBDE >FE GDIJKJ OPI P !! "RN"QR#$#Z2$$Z#%$%[$&\%&','''*]()(1^))+*`]*++)_,,-,b--./.d/.2d0/101f^2.2#2gd3-33ch45i4537ij747k68l9886:: ;<o=< 9<  =; >?q>pF? ?ArAurA?BCvBt@BCCDHDEEwDEFxF>FyxGKGGHzGHHIJIJ{JK{JK|{KGzL~MLN~MSTMSNLN"N~OIOPO{}P! PQ!Q"!QQRRNSUWUTVXVUWXZ$[Zg2[%\\Y\&Y\[],']`^_(_a_)(``a`+a`*+aa+_bc-b,]c3-chdgde/ede0/ef0eff10ggZh53iji5hi74ikjk7jk86lm9lml8km<9non;=n:;o<mon=pyFq?rqsqp>sqrstuAtBvtA@tuurwHDwzHwzwxxwExyyp{OI{}|Kz~M~SR~}O}QPSTTSWVWWW\YZ[\[b]b^^ff_^acbegihhjlkjllmomqpqysrsutvzxyy|z|}{|        $'%%('*0466579 9:;:      >    >    @  ?BBBCBACDEEGEHGFJLJ!NN#  K!!"P!"!"" "OP#R$#NQ$#$%D&-V'U%'WU'*W'*(+(()),X)&*+Y+*+(X,[X,&V,)&,Z[-\V-.\/0010^/0`^121`02`13;3ab34a3434ca45c455dc567677d678d879::e9;e:;hg;bh;3b< 9<= =  =<i=ij> =?jk?k@? >@ ?AmCAlmA@ACmnD%TDFHFFGIHsFHIuIItuIHJMJsvKrOKLwLMLxwLLKMMJvN!POzPO KO" R}|RQ}R#QRS$SR|T%UTpDTpUTVZW*YX()Y+XZ,V[X[\.]\]/_^_^`_/^_]`cadcd56ege;gef<e<9fi<fifeghhj>=j?>kl@llA@lmmnoCnnmoBCpqEpEDqGEqtGqprzOrKwrzsusJFsHutIGvsxwxxMvxLMyNPyQNy{Qy{zyP{}Q{}|~~|S|~pUWUWV\WYYXYZ[Z[\]]]__ifbabhbcghjikjlklkonqtqtrrwuutvxvwzyz}}{~|~             #     !"  ' ' ),+( ./0/31123!  78 9:!6!;"#"=# "$7$?7%&%B&%AB&' & &BC'C)((*D)')***)E*(+,(D,(-+F-+../H.5.G5/.0I/01242M322345%7 87Q8T99 8:SR:9S;<";!6;6P;PW<;W="<=Z>?@\?Q7?$@@]\@>[A%5AN^A_BB_`B`CC'&CE)D*EDaFDF,EC`F+,GHbG.HGN5HIcIdcIH/JLeJK0J01KefKJeKI0L13LJ1MgLML3M24Nh^NihNGbNA5OP6OjkPOkQ?\RV:SqRS9UT8QTU9UsrVntWlyWX<XyxXWyXY<Yx{Y=<YXxZ[>Z=Y[}][Zz\oQ]@[^___A^`aEa`aDEbHccddfdIKeffdKfgeLh^iNbjmklkjlPklWPmmnVRnRpo\|oTQosTqpRqprqSrSUsrsosUTtntuvwxyyyzZ{{x{ZY|}z}[z}~~]}~|\~\]h`_cbcibdeghihiijkylpnpo|pqrrstuuwvvwz{}{{|~~     #"! &&'%''%())*,.545676889::;<>=)?@A><B?D  F  F EF  G& HI J   L    QPSTH XWVXZW]T\`bdd$D  e!! !"!f #g"#$i#$#%%4&&k''j%((*(+)n*)+,+(m,,+o-rp-sr.-.ts.,q//0O/P0MO0/w1M02yz21x23132z3N14}544%j5~7576767~99:9:9;;:;:;<<;<=>=n)=|?C?C?@@A@A@?ABCB<BCCBDEDEDeEFE FGFG GkHTHIHUIJ JIJK KJLKL KMOM1NNMP/OPSQQ`QRRRSSRUUHTUVVXWXXWY^YZZYZ[Z[Z[V\W\]\]]]T^_^_^_`a`a``Qabbcbac_efefgf"gfe f!"g#ih$dhhi$ihij'ljlk&Gkl'lkm*nmom(*n|n=|oqo+mpuq,oqtrs-.t.qtsuvpvwxwvxx0wx10y2xyz}~}j}4j~~5}6786898:>A=>|ABCEDeFkGIIKJOMPOPSQRRUT[V[\WX\Y^[]_c^accbbhdehgigi}lmnoq{prrutrssvuuvyx{zy|~        " ##$%$%#&''+,+--012130353657.798:-<<=>=>@:?@AABBC,E    EF   F    H   JKLLMH NKONRQETSUUTXYQ \ [$\ "]!Z^!""!^"_]#%$$a%&`b&#')(&(bd(e)())c'*/g+;+,C-:j-2.8.*/ml/*./km0n40412o102-j2jo23p5344q3676s7658979u8:vj:;;w<<;<w==wx=>y?>=x?>@zv@v:@?z@?A'cA'ABBCDEGF  FEGPGEHMHI I IIJJRJILKMHNONKNOPGQYQRQRRRJSPTWTSTSUVUVVWUXYXVXVYQYX[[a$\ ]\[]\_"^_]_`#%`b`&#`aa`%aaa[bb(&cfdbef)ede(dfefc)g/lh+,hih;+hw;ih,i,Ck.8krk/.lmknqn01oonon1p3qps5psq4nrk8s65st7stt97ur8u9turwxwhwxyyz?yxy>xzyzv{{{i}|~}D~}~|cAcBCBiC{iDGMIHKMLKLMLNPPPSSTWWWUXWYZ^Z][\^_^_bedeffgljvojmlmlpsq pq  rut   ut v z {~~}~ "#"$$%&%)(,-.03*898:;:=<?@A@B99CEGIKJMNONKONQSQUVWXXXYZ[ ^     _   ] `^ _`   ] a]b\WeedefgYfhihklmmm#opq  qs  !! r!W"#u"#n#vu#"$$w%$"u&%&'&xy'('&y()(z{+}~,+~,--..0.-./1//00/001/2}*2333432*4|475567|8::F;<;;:<?<;=>=>=>??>@A@=AB9BCCDCBDDEEHEFFFEF:GHGGDHGHHEIIJJIKLKLKLMLMLMMJONOKPPNQoQRRURQRUSTSTSVSVSVUWeWtW!tYYZ[ZXZX[^\b\d\\d^[__]_ ]_  `_`  a acaa]bcb\cdedfghkhjhjijigigklkhklllmmnnmnv#oQopopqqsqpr ssrt!ruw$vvuwx%wxwux&%yz'yzyz('z{{)(|47||}+*~,~-0.1}2}23456|899;<<?@>=@?>CDBFGDGJMJOLPNRRUVUVeYXfYZ[ZZ  [ b `^`^   accbedigf jij vnkl   nmopottrr   pqpss  t     t    v w  yxx   z  |~} ~   ! " # % ( ) " * + ( / .  1 . 0 1 2 0 5 / 7 8 6 6 ; : : ; < = @ A  B D C E D E F G   H I      L     L   M      K    M      O L           P R  P    Q S Q T   B  W V  V        X Y    Y Z   Y      G  \     `    _  Q  _   b a     c    &  f &      !  h i ! i " ! " j ' # % k # $ # $ g  $  $ k l % # % m k & % ' j o ' o ) ' " ( , ) ' * + s * , + * + r s + ) , * - - * s / 0 y 1 0 0 2 y 0 2 1 w . 2 3 2 { y 3 4 / 4 3 4 | 3 5 < 5 5 x / 7 8 9 6 8 7 9 9  6 : 7 : 7 ; : ; < ~ = 6  = 6 > = > > < > < > @ A @ @ ? A @ B C B U  B C C D D E D E F E F E G \ G O  H  H J I H I P I P  J  J H K  M K J  L N  M N M N   N M  N N L O G Q ^  R T R T R S S T T S U B U   V ]  W V X  U X U X   Y X Z Z   [ Z [  Z \ ] \ G  ] \  ] ] V ^ Q _ `  _ a a a _  b  ^ b b a c  ` c   c e  . e d e e w f d f  d f n & g l g   g $ l h  g h  h i i j ! j o j " ! k $ # k l k m m % & m n m n n m & n o r ) o r o p , t p ( , q r + ) s u - t , - t p t u t - u t u v q v w z w w e . x 4 / x ~ x 5 ~ y z 1 z y z w 1 { 2 3 { | | } | 4 } | { 3 } 4 x } ~ 5 <  9  : 8 8 7 9 8 ~ > = =  v @ ? ? A U C D D \ J I H P I J K K M K L O O O ^ R R P Q S T S U W Y X Z Y [ ] ^ b ^ ` _ c ` f n f h g h j i i l l q p v u s s r r u z } x y { { }   ~                                ! "  " $ $   &  ) * * - , . / / 0 1 / 2 3 2 4 3 5 8   :    ;  9         = ;   ?   @     A    B ?  6 D F       & H      I  J I  J    J      L J     H   M  M   N    I        O   M O    '  R #    G  S   R    U &  ! V ! W V " # " X $ " #  # Y X $ Z % $ X Z % $ & U ] &  ' P Q ' ( ( ' ^ ) * ) b * ) a b ) ( * + + + - , - d . - . d e . e 1 . - / / 0 f / 0 0 1 0 g f 1 . 2 4 3 5 3 4 h i 4 3 5 3 i 6 j E 6 7 7 5 8 c , 8 , 9 :  9  9 p : 9 < o : 8 : p n : n 8 < q o <  ; < 9  = > ; =  ? > q ; > r q > u r > t u ? B v ?   ? s = @ A w @  @ C A @  B  B x v B C x C B C w x D F D E D E z D E 6 F D z F G { S H K  H & ] I N  J  I K L  L   L K M   N M  N M O M O P P  O P Q P '  Q _ ^ R Y # R T S T  S  G T T R  U  U ] V U W [ \ W V X " # Y Y X Y R Z [ % Z X [ Z \ [ \ ] | H ] ^ ` ( ^ ' Q _ _ Q ` ^ ` a ( a ` a a b a ) ( b + * c n c 8 n c - , d e d - c f 2 / f h 2 f g g 1 e g 0 1 h 4 2 h i h h f i k 5 i 3 4 j y E j 6 l k k 7 5 k m 7 l 6 7 m l 7 m l m k n o p p p 9 o q < ; q o r q r u s v t > = t = s t u u v s ? w C @ x w x v y j y z y z E { | K H | ~ K | }  } N I ~ K ~ |  } I  J L   L Q N } O P S { T S U V V W \ Z Y [ \ | ] ^ _ _ _ ` d c d g e e g k i j l y m l m p n p q r r t s t v y ~ } ~                                      ! " # $ " $ & ( ( * & ) , - + , - . / 0  3 3 /    4    6     5    8   & 8   5     ;  : ; =  <  >  > ?      @    A @      B    C     B C   D        E   D E    D     F     @  G F  H      I   F I   K %    L  ! ! N ! % N " # " # P " % O N % !  & S 8 &  ' ' ( ' T ( ) W * ) ( * W S * ) * + ' + + U T + X U , , . . Y , / \ 0 / 2 / 0 \ ] 0 1 1 . 1 0 ] 1 [ . 2 3 4   4 6  4 9 b 6 7  6 b d 6 4 b 7 c 5 7 5  7 6 d 8 9  9 f b 9 4  9 e f :  5 ; : ; h < < ; < i = = < >  = >  ? > k ? k l ?   A ? l A m @ A  ? B   C   C B n D o E D  C E M  F   G q F G m r G  @ H J  H   H I s I H  J H s J B  K Q % K  L K u R M L  M w L N # O z N O % Q P $ " P # N P N z Q y O Q K R R | { S & * S e 8 S } e T ' + T V ( U a ~ V ) ( V ~  W ) V W } S X Z X + - Y Z - Y [ Y - , Y . [ Z Z Z Y Z X - [ 1 ^ \ / 2 ^ ^ 1 ] ^ _ \ _ \ 2 _ ` ` 2 3 ` _ 2 a U X a ~ c : 5 c g : c 7 d c g d b e 9 8 f f b g ; : g h h ; g h i < i j = i h i j j > = j k j k > j l k m A l m r m G @ n B J o C n o D C o o p E p v p M E p o q G r q s q s I q I F r s t s t J s t n J t n t u K L u w u | R v x p v x w w M x w u L x M p y z O y { z y { y Q { Q R { | | u } W  ~ T U ~ V T ~   W V  a X [ ^ [ ] \ _ ^ ] a a d c d e } f f e g h i l m l o n v r q r t v | v x w x y { | }  } ~                                                                  "     ! #    $             %   &         '      '         (        )      *  * )      ,    ,  -     ( !   !   ! / # "  # "   # 0 " # 1 0 $  " $ %  % 3 & % $ 2 &  % '  & ( ) 4 ) + 4 ) (  *  , * , 6 + * 6 + ) * - . 8 - !  - . . ( . 4 7 / 1 # / ! - 0 $ " 1 : 0 1 / 9 2 : < 2 $ 0 3 2 < 3 % 2 3 ' & 4 5 = 4 = 7 4 . ( 5 4 + 6 > 5 6 5 + 7 ? 8 7 8 . 8 ? @ 8 / - 9 @ ; 9 / 8 : 1 ; : 2 0 ; A : ; 1 9 = B ? > = 5 ? C @ ? 7 = @ 9 8 A ; @ A < : B C ? B = > C A @ DATA-END ATLAS-IO-END eckit-2.0.7/tests/geo/eckit_geo_cache/grid/fesom/3bb8f659e895bdb7bdd60156ad96879c.ek0000777000175000017500000000000015161702250040701 2bdc49d97a27e389fb86decd08a185c2f-3bb8f659e895bdb7bdd60156ad96879c.ekustar alastairalastair././@LongLink0000644000000000000000000000016600000000000011606 Lustar rootrooteckit-2.0.7/tests/geo/eckit_geo_cache/grid/fesom/bdc49d97a27e389fb86decd08a185c2f-3bb8f659e895bdb7bdd60156ad96879c.ekeckit-2.0.7/tests/geo/eckit_geo_cache/grid/fesom/bdc49d97a27e389fb86decd08a185c2f-3bb8f659e895bdb7bd0000664000175000017500000014453415161702250031440 0ustar alastairalastairATLAS-IO v`g \yamlnone: METADATA-BEGIN { "version" : { "type" : "scalar", "datatype" : "uint64", "value" : 0, "base64" : "AAAAAAAAAAA=" }, "n" : { "type" : "scalar", "datatype" : "uint64", "value" : 3140, "base64" : "AAAAAAAADEQ=" }, "latitude" : { "type" : "array", "shape" : [3140], "datatype" : "real64", "data" : { "section" : 1 } }, "longitude" : { "type" : "array", "shape" : [3140], "datatype" : "real64", "data" : { "section" : 2 } } } METADATA-END INDEX-BEGIN |`bnone:f`bnone: INDEX-END DATA-BEGIN mR@%[zR@ KR@CI[CR@gTR@|5R@soR@_aR@[)OaR@E?jR@bR@ϋMR@Cy*R@>IpR@ һR@>8uR@*:R@R@tsJQ@TFLR@6]xQR@ɰc!R@`Hl-R@ s+R@gQ@_daR@Q-BR@Q*^3)R@21TR@/9FoR@FHQ@܈aR@:*R@MʖR@7~eR@(R@!6~kR@HPR@)5v:R@"R@Ӱ R@R@>>R@ }S@bR@1R@uR@82APR@n,R@6R@R@ tR@;oR@(" R@I R@aJS@AmmDQ@z7Q@KPR@o)R@H R@sqR@pQ@QSQ@y*MQ@IkQ@ R@N;Fó4R@Z0R@Bi^mHR@!Q@_xQ@bNR@LR@TY4R@%SBlR@"޾7R@ÃR@5+yQR@%8R@pe$!R@͕ Q@dR@㥗$S@S@8LR@>O?S@N S@h&S@u3:S@p}R@R@'kR@F,9Y`R@O)DtR@=R@W_@R@cR@BOVS@f]iQ@yGQ@I9Q@@ *ZQ@]ڑQ@SQ@yLQ@B-Q@YJ*Q@4OQ@XQ@',R@đ(Q@`Q@2Q@/wR@%PR@JqR@zR@D=R@yXR@1n0sR@129R@]NR@p.Q@2w:Q@v.Q@Ϳeg4?S@ 3S@[_S@]e S@BR@g S@e]u[S@6ivS@s[S@]wBS@ă(S@.l`R@PWR@OR@ń-R@]B>R@!X3WR@йtR@4vR@,vR@A:wQ@cA{Q@CoQ@ۧqQ@jQ@B+oQ@2-|Q@22+Q@Q@mm#{Q@=:Q@Yj{CQ@kVQ@pQ@VR@^A0R@Y*R@CMR@GSgR@c%=GR@Χc R@N[Q@D̺Q@:sQ@R@;`S@ITS@eD'FGS@,:R@|ZS@{LS@1F{S@?,sS@MeWS@T0 S@$o)R@8 2:S@Z߸R@@R@~R@$`WS@]">SQ@8WQ@œbQ@[hu}Q@w.?Q@獴UQ@SUdQ@3bQ@n8פQ@lQ@'Q@jQ@[u_Q@S#O-{Q@^Q@R@Z^uqR@(mQ@@~FR@B4[R@r5^R@ЀY%`R@vR@ER@pS@ jR@RbS@ [[VT@]S@WT@ZճT@G#S@ S@IS@Cj"P@ZQ@@JvP@-<δP@}&dP@wURP@9 Q@5E,P@+L Q@0U@ P@kyQ@`RQ@o Q@wcqEP@,MBP@P@GaQ@= P@ዣ07Q@w Q@Qn R@d Q@q&QQ@U&aQ@`GQ@fdQ@OgR}Q@銥XQ@,@#fQ@[_4Q@:nQ@* wQ@QQ@QP@UR@zR@>{h~4S@9QR@sUR@GR@a) R@ИQ@XmQ@:]ֈS@P/T@ت[<T@Q'tS@8uS@/Gl6.T@/T@wIT@kS@/V_S@YYS@ rS@wP@ÕXP@}P@,Q@斱TP@\{h6P@@z78P@D^P@MѢP@ht$4]P@dqP@9P@P@ 0pP@oP@>YP@HiP@4lP@P@jP@-kj2P@:$s4`P@ezP@XwM%Q@p mQ@_!9Q@\Ea:Q@(SyQ@=eoȒQ@Ol8Q@'SQ@#JP@-Q@ϞhjQ@U'Q@؞z Q@P@w'mP@Mw[R@Ɯ8S@j sS@5XiR@$\R@p7;R@?R@eQ@cS["Q@2?IS@ZIT@v7T@}T@J5+T@JW 'T@[DVT@JX4fT@vӂrT@v+T@SvAT@ן!T@"47T@bt9ST@䌼}T@_2P@PP@R}P@UP@Y& P@{P@X O@6 P@"+)P@P@^UP@(P@6WUPP@R P@eEKP@3ߩP@%Y$P@r/P@ojP@( =Q@DGQ@cP@t:ѣP@0OcR.P@= P@P@׾%P@ZTP@} ғP@SIP@٢i2S@|HSpS@DXS@G#S@:({tR@q])3R@K|j̖ R@VJ Q@Q@]bP@.S@j̨S@u|T@k (T@s˲FT@3[T@~mST@Կ^@sVT@eL|T@҇T@Y|T@`T@CT@x[yT@^T@:?1WT@lq}T@^ߠT@OO@0ǃgO@9T[äP@RxP@x?JP@L4fP@kXٍP@fQTO@i-O@5%P@g`P@=eP@<ϑP@a+evP@eP@%cJP@4 ^P@(\v%U_P@;.kP@lErP@i P@1VP@(+OUP@8qL^P@0aPfS@'/.S@P;S.S@71rR@]CgR@x UQ@{C+AP.R@8_Q@{'CT@,kS@Wj,T@ S)T@n6iT@QȦ}T@^T@DF T@4ƯT@T@@|T@ T@,RVT@hT@ [\T@%7kT@+0T@/T@hT@y!O@vN@ަN@d  vP@6sBP@^׫P@5mP@K'sP@l /oP@ eP@ OurO@$O@bP@P@} wb\P@렖MP@%.2WP@͚-P@vgP@'aI&P@x%J@b1hP@O^C,P@892EoO@ P@i#8P@+;P@ܙO@1Y`O@|WoߎO@R=O@rDL@#vK@`I@}ѻO@A)[ O@PO@PO@9SVO@[<O@ǩ5O@Bp"S@㔱S@jh7OS@kDR@kGR@i_R@G_WT@pO/¬T@#ZT@I0eT@+:iL'U@S"8U@^T@F)T@6}yh#U@3Ol @U@$q^ U@яU@ƁU@r AV;U@+3FTT@mT@ʻN@U\N@G]dM@pB*M@g|P@i8UO@֫O@ cO@qO@EӟP@~P@VjO@ޝ\O@yN@ KUN@BJ?TO@6JFPM@=nO@NYO@#FJFO@-P9[O@LO@1-O@3K@Rc*O@jN@)(O@h=O@|9R]O@".) O@*?S@ tT@)$WS@LR@]qR@\CR@gnQ@3 ~U@ xqT@EUT@`ҌT@P8UU@njU@E: ;U@_/eU@?U@K?9U@"XDU@;tlU@دT@lT@ k!T@MM@ =0L@PtٿKL@vXWO@| O@΋nN@ z-.qO@ oZ"O@qO@:хÄO@'9O@=N@hmJ>N@7{N@@xM@2EO@ ˒*O@]uO@zRN@VkN@>wӽN@=N@[{~M@uIL@ DWL@VN@xSfM@5[uN@e]N@,,|N@E`pT@([%S@ S@?;KS@R@o0/R@g!Q@zZU@lHU@rT@RU@ƍBU@AU@`ժ0U@WWU@V;oU@-HU@?NaeT@ "&U@;DPT@&zT@LqT@[L@18}L@_K@i=ZK@)M SO@>藐WN@ⳭN@ =N@QN@$5O@hoXO@/M@a{|M@Z-,M@q#M@h M@uȬbN@,N@߀N@g[SN@!N@*%YN@;HxuM@1SbTM@,S#pN@!2aN@ 6N@0oT@.7T@d.S@vCS@#S@쇷uR@nE U@V9U@.[;U@U@GU@qYeU@ dU@QyU@ oT@5)S@d mT@5 HU@^T U@gP T@s݇T@m lS@.8 S@C"{YS@( S@(S@OFXR@ݶkIJ@[6BXpaL@S,QK@ K@ً.N@)>;M@C'N@u߾aN@ N@mťN@QarN@ M@vMG#M@lM@@]&M@UCBaM@Ts^M@ 5L@_L@V+K@?zL@ڇH\M@NE8 M@L@f/N[L@TL@OL@% $L@[ET@YBT@]_T@.=7NS@[dwS@ܨe S@RV@n#U@dsU@FCV@r5p[U@ܽL@L@, 7L@:U L@tMK@gK@fzXL@eqg L@pqGڢU@P4wU@xBmT@ igU@B+T@گPGT@ST@$S@OS@-J;MS@9S@ьnRKS@SR@#=R@SR-S@=]aaG@cG@:]I@-ɅI@^ CH@hTG@)w:G@~5QJ@G+JK]9L@ K@cJK@]dAK@0zJ@>y7K@?YN:K@+4K@AQM2K@eT -K@1E7K@PW1K@wE@ʋL/D@ij0C@kD@-fC@> =@ ,W;@MJsT@z1T@WS@SS@6>wS@/%S@2#ՈR@׎ګ%U@;cU@R2s`T@H"T@zu:T@PY5T@YS@uwcS@a}S@t,kr#R@.vڞIR@MajR@ÃETR@&{ F@qF@y6RI@C2VF@LFH@͊lG@C@F@[6zI@wK@)sGJ@U1 J@,;/o'J@2HiJ@0ISJ@MVeJ@|VJ@ 'ӸI@aNP7J@jD@əA@L$'D@qObFB@p1B@s3zC@2B@C@@ES@@U2=@5*k:@ǭ}i9@~& 8@2@)K>T@7S@2̿S@zS@z3WS@"zR@KT@v UjT@y}MS@TCT@5@0T@2܈S@f7!S@b3S@k{R@"%R@ˤ]wR@b*bE@ǔ8aE@OQH@QY-'E@f5;G@@YF@u*dH@&*{I@OlwI@܄;H@$%6I@ha&@_ @@Ve<@I9@;7@o:47@kU5@sӨXS@x:S@* S@ATFS@X T@S@DXtS@(dlS@3`^S@5TS@7+S@s|xR@9>wR@=h}"R@s,JR@|u]9D@hD@\B6wNH@0@4-JS@gd@9@$)=@5}g9@]m9@g,X|7@)q]4@?3@:81@-]'R@O~AR@D4R@rR@0?:R@v[pR@,|R@`JlπR@Q+OnR@9R@t/XR@nnQ@,A@'rRF@식B@R^nE@6K)E@gml D@lЏD@1^lE@%~`F@rDF@aE@.W=@!3?@B;ur9@xy:@*n6@qC8@ !66@Z- 4@U6jZ1@x.@*8@&@7 kR@նR@BeWQ@p[R@?7R@%-DR@`_!R@e.R@+RzɨQ@?Qn@@꛰eD@=1_A@z(D@ϟW8DC@U!YB@m E@~KD@Al<E@Rj[`:@'g<<@yh!6@ns5@Hm$Q5@Ȗz3@4m34@$?2@&1@?D?f-@s M"@#iQ@Sz]Q@qQ@Q@=BȁQ@% Q@yK ?@\ #5JC@ @@7gB@dO[A@̒C@:jNxB@է;D@'8b:@/C7@jHc2@؇1@aZ1@ެ1@s90@׫]Z0@ں(@VDUQ@NKP@ؿNP@FAKP@5p8@{y<@A@;>@3By@@A?@.{zA@BdgB@fp 6@za2@*̓G]-@F͙-@i{,@n;2,@rA*@lI&@;5@[lP@'gM@iL@$5O@"O@R"O@uͭM@Cϒ4@i&7@%߹Z;@&C#y@@(Y<@myٟc>@}J@@Iʈ@@giA@1U1@d9>,@%@%`%@{a&@$J%@ cR@ aMa1@ޫN@(L@WdZU+K@~NCKJ@Y? J@'FN@R2M@#u M@'M@&6K@+IL@yr,1@3˷k5@rik8@B_"=@.w:@ {}=@7f h,@!D+(@Z!P"@;7@/ X@aON@4lΠ%`@K.Rj''+@m' M@dM@Jo9J@ )G@YC5gH@ P-J@"xI@HH@)ZM@(.>L@ZWflL@?L@>[K@FzĦJ@gJ@qd-@mo2@]6@S8@C~0$@Xu@VtҘ@a@E'2W@wK? &@0"@ti6n5K@R4L@4L@LB9RJ@ҎS|~F@庡I@pTMF@jEf LG@aNG@w]}H@ĢbE@J'B@ZQ<@B"%@*LK@WcK@T5J@(J@rhpI@ގJ@;eI@pV铁"@ܓn蹝+@Ώ_22@\ H5@/@ݻ$@ou$?9h+zY @j@:g4s@"]EkJ@\O)L@Nl/\ J@DJ@L*J@WH@=hMVI@M!D@gyG@x1uD@I4E@tPF@jGXC@<*DAA@OB@@x`t=@="5@A #1@|E +@s)U/@ɘq?Íu8@dn!@7mc?mJr+@n{_I@P=I@FILI@]H@G@RH@9@\$@Wx?DE0[#q{?Lu$@Im@_@"?:Sl:I@'H(H@>J@6mJ_L@>[pH@aSCgI@GTG@3X}G@Qˤ!G@B ^C@YhF@4}/^E@0GC@:{D@.A@3q@@e\9@RdJw>@Z<@³޿j9@<9W4@8er/1O;P)`@DzJX[o@@W<@WvH46@+Ǟ:@lex8@7S2@l틕T/@ #+@fC&@+Ѹx"@AG'R @e$Wl ɿLA ^LWVˁ!v @:~h?k0)F@Y'E@SD@XNvC@1C@KA-n6)@KtG Ij?稂+*jBK ӿ\%bk)A0s.TN7Ns$/88<]tD6dѮ!6B35?:qD@u?֟F@ZLD@.ǎ:D@$͍D@ݹC@bB@*"+TB@$ Ŷ?@iIA@{1?@NNč!#60 A.MW2,9wq>IlCU\(wAd{kKe݁tMD&:N"IsXO8AP;@kG8@XdG4@=$!66@:M0@hF*@{($@c*3E@<@hzyO/6L*(,cv àxj(L.kȼ?j B@ TA@0t DTA@h@Ė?@ܭl mxU iv1$߿! @0CgDINP)NAQ־“3e'>P6@M2@&!S'1@Kr:(@#Fc @HN:@L?n1;cF"hF,N4.Jԥ"Xg!6?@S=@2F:@B\7@0#(T̊$a<–s9s1 9u @+敫MYsN:u LL>JFOmW(Kx/G/WtI,bXo0 0jRe0$zר- 9R>%c 7Y"a4UDž^R0K=CRnON?@[އ E Z/yCkAjAS`A@`ѥf=@k=i?@3G0>@)c=@,T=@:<@/;63@2@.7b xJumH[Pٜ3C5[ P*{4aC;a59 36?tmAde:aEyCEmn?@vŕH:<@w ;@-iv6@z9@E:@ɓ8@49@hj 0@7Ҡ0@o.&@k@6*@"̂%@,1 S<7p(QoJ*GW;u0:Al}C.kIE^Ž9@и.7@;r::7@.W8@M5@|9>ӘZd @ᾂA9J@JmD7Yɫ}F鴰;6@^Ӄ+3@5f4@̯I+|2@l TZ13@p/1@6G~-@/@(!@vMǂ @`P@@?%oiRW-Cz@!SAbzWQWdMQHQhAr0GD+a3KnIj6HP`wN`<ŢfOEYPeH@=&Qm#1!B7`^s(=d/ZbS2H\SԔGSZ?\FG4&5>"l4d"QS)ѫESӛkS1y+SSd&S4<S |)OSڱlAJQ4@r7rT=)}/NFE%LMQ >mQ? (O[}QxPcgR `aRzXPW@짜@kSזHGK4ASK2CCd<&C/G4&iF2b&p2@=Eq;0@fNU-@ kx@/@΋4-@י.@ӂ{,@Z%@׶%@/ez@i6w@CY'Z?tV̿KHD>SZI?CWRqRaQ&.AM/ RDQBbs QY4[QDLWOGOFuJ0tP.N7OsBo՝48CU;vӹ0k@NW5'9TE?"9.%DSV:RB=QR)S;#SMS$+ F=Sm!)wSUS="Sݼ'}2&D{H:03U6;}%;J&6SBq)WScF6SB0-SvR8_O:B@5lCo<*Jg LqDCE ;nHN>,@'+]&@{KZ'@s;ɐBY&@%X&@"v;56%@'%@#*>@6HRJ*R#&Qt59Q?8±6_LFՀP'L`{IH8O4SM҆hwDiɀ?>C>gS*5BEY< >7VW?Z:SHR\WRϊ4HRNw^R=DRPPRBURe;[)R7+R!a7*`5r1 *6A: "ފ>:g@S60b S %R/HMSk;RRqR NXdR$w\} Rۃs\?@*!@bwG%@tH-1@ރߟE*@3nv3@,({\c|!\$). GF[2@{h*@mGݵ!.a`A\BRNIHIP]wFݵPN9.R#k&QDE JQ-^R[SLC@`3GD~E#>FKtP)MpGu 0J8 b=A@:ixw@43y@)@|@ ~@F7 @IP@D!5@yH,,Si8#N5 0!>'$!"ÈA]ԃQMQ!NEQ^a⋶PCW@uG&gHߢQPSg$KlN`FB|FdB%lOE]3A.wA Q2Q>4Ȓ{R#0JR0txJ?kRRĐ @R!WntRR@]3&-8ޜ!N<xA֭;i @G3BsʂR1}BR=THRbJRذ#qRX)R*O$Rz0W33g%?b鮲#@Rs@@D; @s?Kz*@Xr$@qM)@r<:P9#f: *xa0;Xh'@NIb#,@y 4@݂W!@7>'"@D3u @<'4L⬈ CzS'A]}DufxO Pq^PT!cEQiPc`RcnQ5XS֮LEHdиN@?GUNuLM|9\J*c@ ![ @ bq@c @ڙh῾ p227l=2 =E. &zS_}*3&scB !QzYg]Q߱ Pr~-Qb P݄ΥC 3I{0)>N Lg{.IHz;jFffC%C9=sfQF^NRwѾR{qH|Rr1HPRRS_eRKVvR.<_6S:7zBfo:ūIC<9`?rA]d@ BWCDw{C풂Җ))Q evQ?Pj*AYO~nE$U3KgaM^fIIGr}El:ܲQaiCRޚQ__0Roj=! R8R_149fl<7c=Z>d\@b`%8XBz zD#D֥XREJR"]Rp:R!"fRZgR7fC 4[ŠOr_ll?(Zɇ @1U~9@"ӓ&= @w!jb<-C2,@^P0&@MP?MĶR ?f?42D%?Oy2Zr۝:E8uy>:=д'A.fC,XE4G R_uOܸ$PEQsPDPH6QhQf*\I]WM%kGچ,@ tHژ@YBY-&uD ,gFe#)LIa^Ok]Pzj$ QbPM9Q,^nLFsNs6(`ra!\(r"_n J,%1Xp;<_=2HM9YC6AјGZ3xPaPo~Ob>(MP96ImK̿e-I szQ-xU Q9ÔQf]/DQa` 7|QD ?;߫?@f0EUsaEȷLAuBeѦ0>Cy*aDB2I[Gx?FQ/֠QQ7R.,YqQIRn:EoeG=]9p3F"gg;6,'ZJ0B[4*g]8xXe e,M#Fy=99w -!:p+gg@ǓIm՞B=J}DF>m\'GӒ7K~SC`K'+mcO26p|PV=VPخTQ^KyM5OTV..ZĎ-1‰"+$ h(@(B5ۉ*1o@=J[x;n PH(QP#NLQ M@J/Kp޼.JF#iQ6ofQD.PZKQT3Py;Q{lnA̅\\@u(ATg3GCF2OIF9o1yb1CBNasUD:%3Eq>)H;I8QoP\wQ\-^Q&QȑQZ R叒=P5m@p ]<_?I',Vu# w20F*43۸83\&^ -%s+P5t(:|(E$l0o~4V%rBߔQIgwDqJF0gtXKAMFaM[`)M="K_OP[xPjC/Qiw~N HP881)a04v'x0*l.C?1]s@; sIZOh~klN/ KLgNKfJ=с&Q18 QIׯRP+ "TPXaP_>MQ2WBA} B{H;~oG<^G9G_yoDX*C@?[0EAx#:F&/]I>.Q>WDAQAFĽgQ&$%Q#oBHWA_h?6>5FѤ;ou6amY5<:Ϧ69ׁ;ڥ!<>T=EzJ0nH7YLwePNfTssNOOҍZNcfWOGPQ(1aPPy;8KC.dJBѸGnO{3[KiJ[Nl,L6 mP)3P}bP,Z/Pd DѧBL>+)?grp@ @\KB⪲CEJ<5K9f0Oj,%LhM=rPU4PV'%O6wӱ3PPE]E9I*wrFʆxFỴGHs/XI',ǿJ5PPP =Q"%Q߆{qQZD@䡩TEPM=^ccP$cP]:aBUUA̭·BBEk'kCow{C$ΎBdECBFUC~[7ODt̙LD;>LE^?H^E2W:F/G7BGꀘqL~.kN)M@ P *}2?O+ZH!PiGu/ŏGwY9JhYHoȏIw׺K$NPi[BPPrM$P~PMUyP8@PL+0Q+PKD+NОH+{\F_KEx8)ErKT)fRVM;5UL\D "Nl[s˷/P wOH_tOK`'PԞ/IgPC6[PC}C}'vCDDnK E=Cz*ۚDXȄEFMFci>jF7( GNj;H9dGcVLVNzbOUjOO9XAHvW4HsJ|'AJnFcLP0'!fP<3 PzCy4Pv!NNP4^|Qe?P(i4P_:F&vIK;2MKTGHH"!3GQI|LqSM+NHM 3(OYM:>[Pvx$PQ]ODEiP]b4DBQ=@tL3Vr%EF/ՒDчnE PKZyF MnFR{miEw_FF5F(<WG5*ƑGTbrH7VeqHnMזMgpN;zO#JS-tFIΈ`CK:^/Lj@[wPJOhEQPIP^nxP;Q!@P8DP*wGzjWJH^WI?TrM7N}-N+AoOs9oiP8%()8PmrP:kuL \MzFBzFL,FꆖHl[\G-JnFfs!GD8^hMHViIO*H=ҘI`f N@ N_Ka(u٫JՆLQeRM]yj%O)*.O5ΉyPcP0IIPAdPB9SQ*_(PPMMbIF>uDK]HMGˁN7HOa7wO:.P|"P4\ǫN*xMV[G/#IJmtIs߉YH"/)IwjFI)FJc*Z K^ANb,Ke!M mNKRmO+ `PW&,PA{vPjjVP 1Pa PlvP#]PENM KZ*N9cL~9M$, ԋ?NH&T~OY`O!"PJڅ PRPh3{Nj9iIe7:J/DJO)eJXi9mGK*rALG-N},M=M}-N3OnO.t_m*P{w.OP"u_P)ݔP{P[Nt#8L.~EMdnfNajHNL WNO iPu9PvƔOBuUKyLcTaKUw72L[gMٷND^O"O,-|O]PeP.P#8^L#g]M&O8M ]DM*NU:N!VN{#1[O?yOQPhbӭBP΢T/PieeP #ݨP.N$}{Sh^Ou2P/m8,Oܿ!PP^PjYJ;P"N3\NqiNuS1OV/OK; ʲOW5PE)P뜮EPO~^P{ejP-p+]OyOL20zPll|@Pجex'PNPdAy O-FOTGzOh!KO(5P KPu=TPF݅P`rsPg]P5uxeP|G_P%-ՏPuPݜP DATA-END DATA-BEGIN Ⴭar@[Qr@D]|r@,0r@avr@gv'kr@ԍ r@!%gr@rΙr@ r@r@xr@.zr@u r@{эbr@_LLr@L6yEr@G.r@$*r@k\r@UEWcr@hCr@@WjKr@Wckr@zir@hfr@rr@޺Q r@hwHr@K擪r@6xșfu@$VyJu@y,r@iq@Vuqar@©j݁r@!5r@hj#r@o=r@88Sr@\r@(kvr@`Cr@w2Bu@h $u@u@Ɋu@`0 pu@̰Qu@t"~u@y pdu@+b}Yu@pNu@Ah5 8u@Y5Nu@Xu@&zu@ Ju@[0ڔu@.Htu@ "u@79|)Tu@(8'q@xmq@fffffq@2iq@6q@1Aq@Lp@8@r@'r@PQiOr@;"rr@Rfrr@;>r@+O r@iYRII-u@gk36v@y.@u@.4"u@Jzu@3u@U `Tuu@35dhu@dLu@ Ibu@w6RJu@E6Fu@ne.u@ykv@lJh Nv@1:9C@ Bh~?{)u@нݳu@G]u@ Bvu@0pJXu@q@9шq@{Xq@R>q@Irq@JH0q@q@:2r@/;=r@gߦr@?C~r@E6`ar@QIBsr@u+:H4r@Tr@vqVr@U r@Çmr@{ŮHr@݂s@x')u@ ?u@5Hwf u@j!?;wqqDv@Ղu@Sv@n,Ru@D)ku@Xu@$)Fu@Ghu@cXu@~+3@K{u@yv@r".Nv@@!x|{?l/ @Gu@%u@6\?Xu@Ϸjtu@)lWu@iu@19Zzq@Rq@ZCq@޿q@v>[q@ ojr@kdWZFr@WYtr@tr@ښHNr@r@.RQr@q0@Gr@P r@5r@Kv= s@p N+u@}fZCu@lƳu@2 u@oYOu@k @U$q@0_u@8@>bOu`v@ChNv@#ju@ceu@0)u@^u@#Ƚxu@ {@5;N9&@7U|` @팥0u@[/4v@ODfkv@!@+IJ@.II@u@%qu@3CX|u@we Su@_@"u@kթu@G 7[q@N]sq@Xq@JȎq@qlOr@3[r@ór@ &B^r@:nSr@p Nsr@e r@_r@-r@Ljr@ s@3Ru@ V>u8u@;2Lt@^%)u@ Vju@7Zcu@YCJ'@ NҀ"@ \\@!kyu@ BK @Mlrv@00v@XAg-u@!_yu@'͵u@~u@!|CP.@L-@mDA:%@pY*+@ٛsKG%@܇ v@A|`'[v@%Zx:?=T@#V @ɘPv@6H|u@`Wu@-u@[Z u@1%q@hìq@5q@",r@yr@f_r@_zr@`{vr@͝ r@͓r@~r@Is@Зs@XBt@(CAu@k@)t@mJ:)u@M%u@18t@ Vu@E_!u@ 32@C4t40@m)@n~h#@d*)6@G9@>@E@yu@'Ң?ӅEv@Z v@j2 u@ju@O1@*PJ1@'X-P+@i[0@Œ,@[l5@L(8@5n:@H("hQ)@  ?v@?9b@@: v@yahu@̍u@9rIu@gq@Gq@gV[+r@r4GV~r@Lkt@&r@3nr@Sr@* r@P,s@z֕s@`%s@Y.`t@Ai=u@uoꨏt@Df&u@`u@dDt@Tt@u@tOu@gUDju@ r{_u@ch+3@%䃞Y$@n.@g)@ǰ6@_y5寃;@9ȐJ@=l@/UC?ue u@7a Yv@kjv@ u@u@XhV5@ۭ4@nt82@N5i 8@3癨<@蘘<@hE-(>@D-@@ལƄU@@2i$@KN7m=v@Eی.jqv@m@Z@4tv@hu@r"u@^q@}1Xr@qq@츆`Hr@}y薠r@܊X*t@\},t@ώ Rt@d{:t@~'r@wr@Er@h;:r@(r@Y|T s@=Ih$s@5=Q#t@/3%Cu@]a 6u@`*Gr(u@5u@3_ou@%s}t@V&u@kKDbu@}u@~Vu@M@s݇$@$hA2@r)@l.@26@ ޡr]."@S] @`G \@;83vv@au@}%t@!>2t@XY6At@POt@iAa)t@GɫS_t@:yt@U*r@;<s@Zv-:s@+fdt@ؑ4 t@{t@p,t@Z u@O*@58EG^&@i}@:[@h=,@zv|v@4Kv@sav@"u@Gu@ D@ $E@'BF@eX aI@˙H@B I@{jćI@Wژ]K@~٬8@@̙H>@m(CD@kpC@4Czt@WLt@=2@rS$@6@Ky5v@u@u@ӤONu@us@ Xr@t8Q6s@n8Ns@Z- r@?@r@+r@R/r@B,r@ySr@9Qt@x/Et@P?^92t@%"rt@XLt@}`t@ N"gt@9ylt@E$s8t@32]'s@3[s@Ns@N`ft@d鞰 u@NVvu@| Vu@張2@l 2@`T#@9o@>@†W?0,?}7?MƌKu@av@Wд4v@% v@rXJu@5!RxuG@uOvH@%!yJ@]$uL@FIdK@YK@pl8@+kql@@vBH@S HE@Rvt@tLt@.rO7-@*#,6@SOv@~m;u@_r@l,ьs@EP s@\S s@c $#r@K7q@Mq Ls@`YiRJr@&Yr@ jr@ ]lZ)at@Ìʈt@=Mt@l553t@a&t@cy-t@^Kt@>t@a1t@ǰ Ks@}ts@9s@LI{(Ms@ɧt@eO2>9u@Up6u@1^u@fڋu@jIu@[iY=@~ˏ@p @.GGE?q-vv@_5sv@P첩?#u@rHv@bv@qRu@mXL@BM>M@.VN@מYM@^)O@a=I N@2N@6@|IA@? GL@;ZH@Z!zt@S4u@7\#@.6v@׎=',r@tdr@@1[s@}-s@Vu1q@fq@o}q@F@6s@rߊs@:a"ɵr@lt@8уlt@PRus@~n8Lt@% 1t@{t@Lؼt@f t@P0ps@Ns@hUs@Ivcs@1Qt@Z_;u@9u@_Cwu@)E1Yu@˨`u@Pc+u@诡?t?+oYv@< R[bv@ օwv@u@5pQ73v@L v@(N>=P@(=DP@ZfQ@CQQ@ O+[P@5NQ@NXHrP@ E;@kFG@֬3.P@pX>t@|u@d?#@b(q@w+r@z{r@BIܾ"js@J#op@ -q@{+q@Inp@-p@wZs@Jw?s@,x#s@3r@hzwt@P*t@ivt@,&6_Wt@r8t@p,t@m5|t@ b@s@ t@g.29u@Z, u@Э![{u@ѱCWu@efbzu@صb$u@\iv@+Nv@tu@v$v@C~GqS@ p@Ƕ Gp@-[p@U^o@95N9o@^G Hp@Nup@UI*p@٥W p@ɯb#s@fRhs@(3ɿNs@V#}2s@t@QL9t@ݓ,t@鱲bt@lp@>t@:t@ t@Ls@/s@\w]s@`Os@O(t@g=,u@ȒXu@Du@b &u@ efu@®8v@m}Gv@\pV@8SV@$P.U@fuW@3ZT@B,dU@sT@9)P@+_X@yY{Z@cADs@jp@Hp@*4o@?o@.q^o@Xo@$G:#o@~=zbo@9qn@{n@Z֘`o@to@=\o@j.JXp@,3[h=p@4p@IQJ2s@r7s@ḌZs@Xniu~s@)n`s@PlOs@,Gs@Sms@88o s@:[@h}ns@0ms@~/;t@þt@ֵ1t@RQct@ϝu@Gu@:6Յu@R@Xu@e 1u@4\.,u@Dr@Bq@<ѿr@7r@%+r@c5r@.ycr@5dr@t|6q@2rq@)Xl=q@uQp@Gp@^wp@V)=KZc@ڣ"a@3?`@!j_@S_@=+O`@.(g@xJ}Xe@i;d@ [oh@-t`j@D^kk@\Wȅk@f縿k@? /l@"8m@ l@QYs@՞?s@=/pes@& trs@"fs@s@G#t@1t@xt@+3ASt@ Jhu@C>C8u@H=ku@mu@0u@?u@b^ s@x2&r@[Ɨq@Abr@ 6Br@S7r@ha@ N f@(1e@n@d@džɯi@YcuP.h@JShk@ѺWWj@ Qj@UEWk@g!l@\.#ns@"O7s@muAu@u5Xu@NN-#t@)Qs@cyW}pt@Vt@’}u@'1u@L;Pu@PR v@җIv@g s@^es@c8r@74q@){K9r@$s9r@(< xr@dq@njq@5iq@DlYq@ g@Fރ!Rr@`sr@G r@FFbq@yᥥq@Hf@Vnd@q)te@*6g@ vKg@h@hOt@t%]u@ ut@IPu@yjDt@6c0s@|7̆+s@s@.ic^s@,>r@dsUr@_-r@-͓#r@r75ef@,G#zXg@ya!ig@+f)xh@vt@5wt@ҝq+fu@גt@c7u@:u@0עu@XAu@ rr}?t@İs@nws@s@l/Y{s@<-?p; l@w%k@Jk@XJm@SW~f@0Yg@l2g@5Hh@h@Gj@ {,i@$4 u@w`'q(u@`oFu@[Du@jt@}wXt@UO?'t@`Ms@`0$Ls@ ?xs@ї9u@Yb t@}R bd@z|le@6"f@ѶPc@2l@T?j@΋Rl@6y%$- l@ZkBk@ݐKj@]Lsfm@-m@rdq@̈l q@BMg@72@f@0nh@g@?}pVj@EVh@kApi@|C5;u@?vCu@]S{lwu@ǚ6{t@^t@Eo.t@[,s@*/C-7u@}ot@,2u@!x4 hc@_!c@Γ֗d@*e@N5e@po8d@ [c@(Zl@Ü{jj@n81l@Ԙk@k@0l(l@\k":m@^gCm@ƈDeln@4o0/{o@\q p@-zsp@h`p@Uwq@%l!q@ a5Vp@9 .q@ Rp@Mg@Y`4`f@,iPh@xM1g@Zi@%h@vhsu@r?مu@\6:t@Urt@C;t@qUt@=@&u@wIu@u@=ԛu@8v c@Grzc@2b@H;$a@מe@;~e@zd@Z0c@ Nc@wRcl@[!Q i@S{j@-~k@Ʌo5k@N-Ql@H4l@wP/n@;.[m@>hɿm@E/7,n@dCRn@5 o@] zo@ #bo@Gҿ'p@&s|op@r7gq@vep@aQl`p@:p@"!̈q@M\3>rp@P%5p@9 f@ ݾf@#Mh@dLNg@Qi@(Ru@kk&u@D0v@6nօt@g st@ӂCt@t@s*'u@=u@ΔReu@u@]NΖb@ndc@[YGb@~Xe@-w[d@'>d@вfc@NNc@k5 l@|jPi@yj@LjVk@Ȩr@JHZq@v$Mrq@r@l q@eDXCKr@7Sl@| l@Dn@c">m@ym@E˪>Dn@'=n@ﱪo@jao@tF_\o@E /p@;29q@|@p@ʨ2p@`)ԩp@]<4%q@S9pp@)yup@@p@٩^Cp@{dp@Q4p@uMnJe@f~Mf@L}g@0V?h@D2?rKv@B?F =?7s@XU &C@R.r@<]Xr@ 'Er@ xr@[7r@~ Gr@NVr@`ũu@y,v@c8\[t@S!u@z"t@!Ns@AofNt@ im_t@7utPu@zۡu@im!s@CЇ\s@+L s@}*Ts@k͠zs@0s@pa@e@80Fd@96c@p;b@pb@IHck@2i@ j@o9nq@|,GhJq@sqehq@%4q@zRaq@Erq@D@togq@awQq@6Bbtq@#:ڼq@wr@ r@p= wq@@$r@??tq@Ɣq@.pyk@,#l@Œ)m@lm@ ;.n@'@n@nCqAo@$u{_p@ep@up@?bɘo@G仔zp@Qe@8<#g@uh@n8) $i@w&wv@k@ $M@u|j@3?H`i!@0mr@0r@`Tr@UA*s@#Ss@E"s@-Ij&s@,qu@C}=v@0Nt@%`/!u@<\At@bt@>dt@&6m@Ei@og@\h@P%i@2-k@X}d)j@9H i@`xh@}@lg@s/0+ h@fglg@^gLf@l#g@/0g@6=g@&*"g@H&^f@9>g@Xf@;ye@ϔ ke@ Ue@ ʼUe@td@R{d@1K@A="O@hSK[N@ҭ2MP@ ZB@1%0@j^e)@}|/Qt@U̐t@ Ks@`\;s@VY%t@u)dEt@cs@c'Ws@(o%r@4"@ט|XD @> Й_u@&3Zu@`0$nv@ Zru@i uu@Mu#]@PI[@- ^@q)Da@+#6`@ G `@;(_@q6G)+b@'7@a@"^Frb@Yk(5~b@l~e@Wrc@1Jc@%Cd@Otc@o']d@IOqhg@: l@Npk@l@IÄ,fm@0j@ͰIn@Inmm@Xn@fn@Xp?m@() n@"z0m@VNk@Oݺk@5`+Yi@tih@ ih@k]@k@3ۺ@ni@_ ըj@>j@fqi@[:h@F% h@$ݴg@Z:g@df@Zdf@ѭ|;~f@j&3f@ 6f@L@"g@3OJf@J3f@2;e@foٖe@Me@)3 e@d@M&\d@(;(D@|MjüF@K+tI@YK@ \(N@~!d@4^_zc@=^c@p7mb@"g@1k@{rτj@zl@!u;{k@LRZl@dJ&i@`4Zl@l@TJIl@EKk@jü"Qj@[h@| 2h@nzoIj@_ӃR.i@M:[i@vUuf@{yfh@$1g@J Sg@v'髵ee@ ~f@mo\kme@Ze@*e@cbq-Zf@ɍDZ f@~f>f@^4;H@ WV@fTtPX@xFS@uU@]T@A6xV@nk5@ 0m@@nLC@7#=@,kj<@'++9@2@B/,8v@a4@v@_u@u]u@Hg`9u@2Ƈˢ.@YOJ@p](@ԏ@r]b_@$`@(z`@F`d@}4濑3d@. Dc@ɪ}fc@{)Pc@Ъ;;b@&b@$g@#6j@Š]ϡUj@WM1k@T:X'8k@,e|i4i@H?=k@oP0k@LjhZj@TYi@mP{_h@p i@4@/f@p+sh@hf@p&=h@e*Wg@eo)Kd@dtf@Re@Rme@GMe@a e@G):g@лgf@ h`?_%f@Ie@ /f@x0Ėe@EPMe@[r&7e@E@)\`H@V;L@(xzN@*ά:xP@:*Q@3-(R@ʁðhJ@?[L@ KJ@tm?J@*=%S@1 7W@.WT@.xV@W/U@z W@~UxV@-(gE@zVL8@a¹A@9C@i@@=zqs;@֖^q0@QO1@"@1iB'@#??O 8@_65@)Z@-4\@]@ !_@uz `@ͥ8a@7d@SB dd@ho:c@Þ&c@oUfJbg@cyi@-&i@?q[6i@/M&[i@ꂰ mh@;P'h@g@T"?f@tDWyh@Mf@=&Bg@G2(g@xrd@Śc@F+5/f@HD e@Fd@[Ye@8{uYe@b]ƅe@IF1f@m3of@N-e@Әe@B`e@Me@>zZ@qi [@4\@ _g!^@Β['`@^kc@4c@Wf@x==[h@܌:h@0]h@ՠy g@B11g@#|9*e@{:h@Cf@nf#g@ȣef@B2g@%Oڱe@d8|Ed@3PlEc@>}c@ԋf@/6te@+fye@]ؚd@@ޫVfd@Ee@ Ae@#mf@Ic7e@Q0c e@aqVe@^Bͤe@F0`~e@e\e@ 6lJ@LlBI@6ڰL@ [T@DX@yG|X@ĐAU@U/dW@W&-Y@ƃ-vY@h ;G@ḌD@1G@i\]yF@YF@S/B@*EjE@7;@Yک:@p6@1q@A@8~M{Q@@K[@6xȹZ@5=Qo\@p̲'U]@/>9^@f?`c@Տf@>%g@!_g@(g@(8Y@nPZn+J@G UůH@j#I@,H@$*I@1!撈G@V[J@nYtD@<^?@@ӧ?@<*C@!!B@u=[@3:[@=s0]@tb@I%f@Ad_f@mf@'q~f@铁f@w7f@t12xf@NeJe@D3O/e@@Ped@ZRc@`~]<27c@7ѝrqc@dEe@hPe@96Ģe@.@ld@{mZ d@t!d@AVsd@ad@ز!e@Өv|d@d@d@kh^id@Ӵ@d@1 r M@eO@),Q@=e5]OS@F}xR@qnU@0mW@xb1Z@-/W@GRY@ BX@/iQM@وqHK@~M@ ɺL@i-qZM@q&\~*G@mHΘI@ J=L@xE@eGD@o'D@-@[[@H\@rHX b@vs¤e@tGf@wPe@a aFHf@vyMe@Ld@CHe@g}1YLe@a8{d@:c@qzr:c@+,De@Ao'd@y7.d@)+d6c@L&Qd@ű'kd@+-;,d@Odu|Ue@8)4$d@|r[c@Lb@\ d@pAc@IXkc@:Xed@I>d@x}2d@xld@̫qc@XZc@idd@~}cCd@~d@1P&d@ӗJd [@$tS@+JT@=s{n%V@[.(Z@ sW@!e1"Y@$)&R@TP@pi R@tC[P@^1 Q@"(P@_u&N@M#K@gd[@x@\@ s%^@.[@s_@*`@0;`@{B%`@Oj a@ǘXa@KO$b@Zb@жLb@G<d@ZIe@rd@+¨!d@R gd@YTd@9cc@@')>c@):b@+͜vd@c[c@|c@-$d@qd@[,ąc@}R c@a;Σc@/<΍c@Ȗ2c@yc@j]cZ@T@ r;U@V@a{{|Y@~)iX@]D%S@L泗HHR@SPFS@]cQ@ӈ}IR@;P@a/Q@pʁðN@b=Y\@r^@O]@2r,^@/[@^[@Ĭ_@CK~`@򇰫'`@TĚ`@7<a@ +>a@ kTa@&a@@pKb@@1(b@Ojd@d@q`(.Cd@@'ic@N$/d@rAIc@Lȿ`*c@b6\?b@)ād@? ec@3c@.Ec@iFc@c)Dc@3zc@caUSc@?%Kc@}{c@#c@n%c@!;oc3cZ@WJU@OV WJV@nIXW@s-Z6HY@EKU@^&_S@)ODT@'l?wS@R@!<5_@/A`@ew`@`XY a@wa@1ya@4NXb@nea@6jOc@&2c@c@\Hc@ab@b@}k̐c@,Q .c@S?oc@{_/ c@4c@5لb@yDc@Mc@(NI1Uc@KH'b@ް Yb@: Ai]Z@LV@DV@'?ʑW@Wh3Y@IB7V@<|T@dVU@bIT@EnR@VS@%|rsT@(YS@{R@ )bW@N("]@ M^@^@>*y[@Z\@ysV{_@N%@8`@76`@ XXa@_hua@/ݧa@$1Ƒb@!ec@@pDc@h@ Fb@ób@ T6b@Nb@qb@^b@{b@wa}b@ByGb@4b@9\c@O'1b@9.b@sZ{Z@`tg\OX@) RY@v/W@߼8U@]4wV@=_XgU@WpmT@U@-RrT@8>)LX@:q'X@Gdiq]@p#^@D-[@ŭh\@q_@!K6`@k{Ƽ`@c[7a@1˞a@ab@¡xxb@0CFQ`b@E.a@pi떓ub@=rb@:2Pb@6b@ Ʊb@y)Eb@Mbmb@hb@QJ-b@7f#!ۀb@Khb@U+Z@vY@/Y@ۢqM&W@d=YV@Kd5W@xnU@NrFU@ùX@u˿a@99wa@i2b@@k~%Lb@{rτ.b@ab@%!Hb@j9b@e!a@J2Z@FԸ[@PW@U%Y@%ZMA^@Ra]@*{_@o?M`@~+P&`@xBbVa@ܔKa@ %a@ q2a@ ra@Z-DBa@ѐ(Ua@Xa@b#a@oɷ\@`8\@_;ѮZ@);;[@ ][@矅Y@!X4X@+rNZ@u^@k]@KóDB`@ _@`@6W`@x}9`@+N`@0`@ܣ`@́ԜRa@r&a@7la@H3Ўa@^0C]@Y|T \@Gz[@շ#\@4.w[@`) Z@QZ@!^@{'<$^@j?cpޢ_@߶[s`@dӥڂ`@dE`@ ZH`@X1`@SFL`@4Ơ(a@yFa@ a]@)]@ \Ȫ\@HZ|[@"JJ"\@@4[@ǖ2@^@)yG^@-s_@1Jҍ<`@"q_@Ǟ=i^`@`@9υ`@{7]``@c]@A]@SO\@ ǘ<]@e+\@e$^@wxtHW^@ܒf[_@G(`@>_@mT`@F)`@7x]@;ǀ]@܎g\@ɕO^@0P^@P Z_@1`@-*_@4z9`@xADjZ^@0h|]@g$^@͚S._@$hc_@@^@ _@ DATA-END ATLAS-IO-END eckit-2.0.7/tests/geo/eckit_geo_cache/grid/icon/0000775000175000017500000000000015161702250021647 5ustar alastairalastaireckit-2.0.7/tests/geo/eckit_geo_cache/grid/icon/e8e7db507f0b28dee00cb3f1867045f9.ek0000777000175000017500000000000015161702250040103 2e234e01a8556e9a84bcb42361d2f24e0-e8e7db507f0b28dee00cb3f1867045f9.ekustar alastairalastair././@LongLink0000644000000000000000000000016500000000000011605 Lustar rootrooteckit-2.0.7/tests/geo/eckit_geo_cache/grid/icon/e234e01a8556e9a84bcb42361d2f24e0-e8e7db507f0b28dee00cb3f1867045f9.ekeckit-2.0.7/tests/geo/eckit_geo_cache/grid/icon/e234e01a8556e9a84bcb42361d2f24e0-e8e7db507f0b28dee000000664000175000017500000012655215161702250030762 0ustar alastairalastairATLAS-IO ҇^hgPjyamlxxh64:ca4102cc9265de140 METADATA-BEGIN { "version" : { "type" : "scalar", "datatype" : "int32", "value" : 0, "base64" : "AAAAAA==" }, "shape" : { "type" : "array", "shape" : [1], "datatype" : "int32", "value" : [2656], "data" : { "section" : 1, "compression" : { "type" : "lz4" } } }, "longitude" : { "type" : "array", "shape" : [2656], "datatype" : "real64", "data" : { "section" : 2, "compression" : { "type" : "lz4" } } }, "latitude" : { "type" : "array", "shape" : [2656], "datatype" : "real64", "data" : { "section" : 3, "compression" : { "type" : "lz4" } } } } METADATA-END INDEX-BEGIN Exxh64:22e1f7571f83f2e7 Sxxh64:12b19f62d0f12102YSxxh64:38d68183b0dbb011 INDEX-END DATA-BEGIN @` DATA-END DATA-BEGIN DoO@.O@ЬO@s%O@7rO@nF$O@6:YP@ȤP@IO@7*P@0Q`P@ZP@(P@8` hP@9P@ 1oP@sÎoT@pܴX@WlǨX@rQdW@u#U@H@~T@CVU@95OUV@&V@=pU@L("W@QrP@ uZxO@)#P@Cԥ MP@9!CP@9?ljO@P@@WwS@{`R@ϒ}R@+M"Q@x*Q@8 Q@EP@v0S@' ĪZ@ 4Y@DFrZ@݊PX@WY@MAO@&hO@VO@x[\HO@+_?O@i28sO@AX^P@e/P@ P@2O@הP#P@p O@ CsdO@QP@_"=P@8]3P@aFcO@WiP@{P*P@8P&P@H O@tҟ-SP@P;xP@jl"S@SS@w"T@}sGT@Q)?-pT@e^T@GET@ϺUT@ U@a-U@KEӂVU@ yU@ilU@b^U@M<:U@d(V@`@w9V@v8M[V@܊1V@3.HV@.3V@&CW@}(,KV@=LIW@wv8W@k_|aW@-t1W@JBgAlW@z5`X@ssW@v@uX@b3X@J6tW@b{X@a2oX@o;dVX@A'~Y@Q?Y@_Y@!K4 Y@(7Y@;?44PY@FďX@rqY@A~dY@}Z@-FZ@ ZZ@ gxZ@/VZ@ Z@qZ@rfO@95P@SluP@s'P@啿{Q@8OP@P@bCKP@b\P@c]7Q@5Q@ ʗ3Q@2P@TT@_W*cU@{/T@_W@[wU@ч V@ܨkV@ ̵W@czِXW@zP@0 >J~P@@V^V!Q@SR&Q@jyWQ@2R@"#([Q@Q@W)+tR@z!Q@/]S@Py#S@CGCT@3Z@/FҼ=X@.+X@TY@W؍iY@ zZ@=Y@w2 P@oRP@!@1CP@LWGP@+vƓvP@8iR3P@7=VP@A,zP@AUcRP@K8/P@ 0u_P@C#MUcP@o轎P@"HP@a!LP@&߂uP@ܬ,P@"7P@$SpP@^qnP@@qܑP@7}kP@9RV Q@H>4Q@3aQ@Q9Q@PQ@inOR@2|HR@%jR@wR@ZR@xʅdS@Q;;S@F#xS@A#iS@~T@pS@dhS@䆛@&'T@굞T@{oO#uT@wLPT@͙T@u:U@7^n<U@ 朒T@ҌI]U@!'U@8o["W@<|FLW@GIX@A}ΔlX@Nʫ޼QX@d>Dn#X@7KX@4W@`>W@T!X@yIpuY@_zDFY@ToՎy-Y@G?X@bX@iABZ@`Z@ 1}Y@Y@Y@Y@\ VZ@n5UGZ@hʹZ@ȚVZ@HZ@!]s{P@ߜ>j!Q@BP@T+P@b$Q@VP@V(P@5a*P@uW~ P@*P5iP@~G{BoP@f3P@i&9P@2P@UKKP@{2\@wf4\@\#\@cϳ^\@w#W\@ԋ+o\@f\@G.\@6]Mi\@J^k\@9p[@vTt[@Sfa[@fP訸,[@[*[@ۚi\@i(֖\@f\@T\@%fp\@6<\@ժ,:\@E~I\@:o{\@bEKd\@@I\@w&\@Y 1\@0G\[@VH[@pdv\@/* \@ <x\@}q\@N z\@Jj\@i]d\@ÞW /}\@13\@_pH\@y\@ :?\@z"\@l 46\@9s\Q@[*Q@ڑ_Q@bD¬Q@9YQ@(XSQ@PQ@'|#[@B*i[@кH[@R[@F[@Ԫ-[@Z}[@V9x[@>ki[@[@|p[@N3d[@V[@]-[@xd[@Ӝ[@\@kZ%\@I?\@F\@)e+\@( L\@JS\@΂j^7\@[;S1\@BZ\@]b\@rD\@U˘j\@(r\@-S\@æ8WL\@-s#\@HV\@V\@#\@f \@\@ Q\@G bb\@_wܙ\@n)[@a-rF-[@a#c[@R2]\@z0\@1!|\@%8@\@$ \@D \@±ʢ\@I;hh\@$3IP\@`{O \@k$C[@[1[@>[@Ƕv[@IO(\@_.\@=9 \@78d3\@6GG9\@W7G \@8v \@٢@[@o[@9^Cu[@{qްQ@@vR@- BSR@!R@Q@SjR@Pe+TR@{g쨦Q@n|WkR@/8S@쾈]R@ .bR@ Q@nR@ [`R@$Q@/ޜ]U@/}U@ti&oU@<_SV@̎V@0k!R@><`?S@S@0 T@̉T@&"T@hV@+>/jW@9:ԴW@JW0Y@pY@⊊YX@Kl1 Y@WHY@$f>X@Z@<mY@Z@`Y@9Z@`ΉZ@W FNZ@;GP@6gP@x~EQ@Q@a2Q@o|=[4Q@oH Q@qi]Q@R@p;zS@/uS@ nS@C}CS@IMS@4}?R@W R@z KS@]ѠU@ lT@[v"T@,?T@ r`T@`$'5T@a&)T@uy$U@I trPQ@T Q@#3ԍQ@908lQ@xYKQ@ eQ@q:Q@>Q@>^Q@kQ@|Z@D܉xZ@˶$+Z@3MZ@mm<)Z@qU4Z@l5Z@MklZ@M "Z@wTZ@Z@IZ@:(`ɨZ@H'Z@|GZ@TZ@iO]R@E?R@5g_S@*(QJS@I4 =qS@}NS@{PS@l>4U@d,dT@}%@T@kFeT@3ET@bT@UT@LIU@p^S@lģSWU@qńU@x2b&U@4U@/U@K$V@)Z@hz8Z@*MZ@R@PQ@ͽv>R@N@DR@7JQ@R@_RR@|Q@ tQ@_R@ ER@~ LZR@R@)gQ@Q@kz障Q@|Q@ÜGMQ@6;JMQ@lpOQ@!Q@eHQ@]2FQ@w$sQ@[wrQ@2JQ@4;Q@,WXQ@wQ@QR@~}Q@Q,Q@b[eUQ@V> SQ@w/.Q@:YQ@ ZQ@@qQ@!vQ@8PQ@0y )R@i=B{R@+tRR@Q@N}x%[@.@[@ǚ5[@PB8[@񵑲~[@[@ _[@t[@=y[@B7uu[@l[@j!&[@SuKw[@[@R[@2)Q|[@N0џ[@p[@ǩ"[@Zq[@ۘm[@ ;[@$N[@nQio[@IF"i[@xB[@F+/[@%uf[@hz[@- |[@'_R[@eL[@Ks[@.)[@*>)[@q@F[@/k[@1:L[@O+mϊ[@.™]s[@Z S@ rS@/ S@wxS@9BVT@@a.yS@oS@_)@{V@r\V@2db@W@H.X@3sX@H7EBV@l`vV@#3=VV@(NV@0ΠOW@ W@YhiW@0 3&W@ܭV@jjM$V@dX8V@*%V@..OU@`dV@;ʬqU@y.Z@ NY@KgY@x{Y@JsY@Z@3D6Z@)= Z@^ut=Z@VNϑ[X@l X@.pX@X X@W@4ڔW@e{W@yW@N X@tO\X@>%X@ BW@Z+X@#Y@Иwh"Y@X@FHJJZ@P]Y@X߼OY@) rY@ƛEY@7{0Z@I PZ@dBp)Z@cUZ@_Z@ AbY@*EY@'HY@ӻY@(&GZ@lKtcZ@O BZ@V]gZ@(xY@* M­X@3nBX@Zd!Y@MmrTJX@CniW@6X@Z&[X@cnUX@ hW Y@ʼX@cE|X@XT-Y@N X@6iY@,Y8Y@LC(Y@kY@?ۛY@ŌVzY@lPZ@D7rkZ@zKZ@Y@{ĝqY@7aY@l%Z@EpY@eU}|W@jW@1V@ zcOV@EgV@#@֗V@;W@W@̀KW@P|$W@W@[τ:caW@ lW@1 W@O܉v=V@iDU@x*1V@?XV@[?# U@~WaU@irh՝U@F*U@0bV@?@RLV@M@n V@x U@ӼrV@Ê@(V@0;fV@60PV@F[/PV@}ĹlW@k-V@AͱV@μ߹bfW@(j@W@vuW@ ݝQW@)W@y2V@ͤ 6CW@w$W@d*V@V@THV@!fj/NV@W@ W@Y4zW@ 7JW@0s9W@XV@$c+W@t NW@"xW@W@KKIW@yzW@;'ݱW@| p*W@@WsqW@,(%a X@VUsLX@%X@H6^X@d>X@ X@cAEKY@W}mY@X@&X@Ð'OX@5!=OX@sX@q7x/X@F19(لX@ IAX@gJƩX@rHY@b!sZ@%B Y@Y@'/`Z@ƫEZ@xZ@&V\Z@>XHZ@5%Y@"FaY@mZ@zY@l,Y@y#VY@3Y@n?CY@uqiX@sse-Y@VMY@O\ԉX@$M1X@vX@X@JX@6Mo7Y@9r?,X@X@$~ʴVY@_X@} AY@Z_Y@׌VX@dVūgX@!X@+W@(X@Z;HZ"Y@ bX@S۵X@F@kX@'8TWX@(XX@0:{X@(W@{HpGX@RW@цSW@%&Z@'{Z@ ǨZ@Z@ڹZ@ qY@h~$Z@4Mc"Z@u5fZ@XZ@2=~Z@DVcZ@pOSZ@G2/Z@";Z@oLGZ@uE/hY@L Y@VAGTY@pY@ srX@%kX@^ƣjX@O5X@8Y@ϗ\Y@{Y@K↞X@~uxY@g%O(%Y@Ks{eY@9KY@X2Y@5s߯Z@Y@ܢY@GmZ@knZ@Z@3 jZ@ta&Z@Y@z8E[Z@+Z@fIY@> Z@*szY@D`Y@S[Z@@a{Z@RtZ@hVZ@`,/Z@mIY@~9M Z@]KZ3Z@~ƖrZ@PZ@6^Z@ˍoZ@Z@\uZ@} Z@kI(Z@Z'U@n:_AT@hU@De=U@dT@/cT@Go"T@T@UuCT@2S 5U@[xT@3"h۳T@6W[U@:!ߺU@O>SU@GZxU@'mi)T@lJS@ jf $T@msk#KT@nS@>NiS@S@Ix\lS@65T@7XD!^FT@+T@UTJS@TmT@,!17T@K}gT@{.|T@aNӘT@nST@x%,T@;xF#T@|2U@g19pU@D퀶:U@rU@qIT@fZT@CLkRT@T@psAZ~T@E:bT@~"oT@:\T@WU@[`WU@ط/U@zaU@o{U@kOT@cGIU@U9U@#kU@ %;U@DsU@MgQOU@RcU@1RU@DxU@|U@rSWS@.(S@+TS@A|S@HR@{T5yS@[/MS@%S@,S@ iU@fFH;U@WmU@',UU@SaU@eU@1ƨʆU@!UU@ГU@nFNU@O;jqU@Ų[T@5}&2U@@U@!T@lT@]xT@w?ZT@T@sET@ˤT@ —T@]rT@nT@LDT@xьS@2:%S@vuRT@jjѺT@ơ&]7XT@r2T@w&T@qS@pmS@ۨ$T@ZͦS@1\ S@m KS@^ 0eЅS@CKU@؛PU@a9U@p)pV@ç8zU@Go V@zAU@ RU@X-V@򬭢jX@~5X@= 3X@VW@kW@mW@,xW@EEW@!DfW@6!9W@$V@I[V@E`cY@ QvY@e4dOY@\$6Y@XiX@R3X@rΝX@IdY@?FX@ЎX@P 9X@M ԺČX@,QuY@c p&Z@zY@Y@p_wZ@7DZ@~Z@[F4uZ@t6Z@>Y@x"Z@Dހa:Z@~ Y@p[:XZ@ٹY@~E>Y@T('Y@6Y@ÈtY@+}zvY@C@X@Y X@ZX@MX@xg϶-Y@}{Y@Iģ^?Y@LA@%Y@u'Y@pGY@*PԂY@VY@YX@ wSFX@A X@ktX@>.Y@:JMlY@#g<.Y@fHJY@L X@VKX@boX@|3}X@ЊgX@x&)X@{xX@єZX@5ÊhX@W@D X@?*X@ӚW@ >W@cwW@>͖W@b\W@Mx,%/X@7W@i W@L8X@>H"W@`K)X@i6Y,FX@oV@WhV@H;V@{ W@@?vV@Gr8V@"/akV@?nnV@J;V@ْW@/a-dV@']1V@KW@hfV@WlNW@62W@RddgW@Կ*W@LٳsW@SaGUW@mIXW@-6X@HoW@vF[W@InW@xT{bW@BbۙW@7)S◷W@+SCW@lepW@id/҉PW@j+1W@.0SX@  X@)DX@i_X@:͝]W@K4GDW@ADW@W@0 X@>l QX@.Xܕ"X@|NW@HAlX@'X@*}^]X@/ _rxX@WV@2UV@- 0 V@{cU@EV@;]V@L.4נV@s}GV@UHV@V@f=V@o_V@Jz)U@ڴ9w%V@r}UU@6U@$W@_oMeW@՝s*W@l W@A!IW@{(RW@q!KW@+ FW@+(RaW@3B$W@)3SW@XsW@=V@PTV@9þ?W@_kW@ (/V@9lV@8Z[V@F@kⷖV@?;ƺV@4;V@V- U@ƀq V@4q/V@&%7hV@ ]{V@ǭsV@KIPV@cV@), V@[DbV@JV@[uU@M}AxU@OʔU@xU@hخ h7V@Jx~V@vBV@=xV@O{F/U@}sU@VrU@LmtU@JvU@_(EU@u,EU@lqXU@m'0Z@0q`Z@e yZ@ԲZ@Aq&=Z@ 'Y@[ D*Z@^,AZ@'x|Z@]hZ@n Z@f,#zZ@ 80Z@c4yZ@1mYZ@ Z@uY@z>0WY@]9Y@wY@R:hY@!NX@ۀX@_Y@YR)NY@tY@MJ^Y@RFY@vY@ZfY@|Y@%N5Z@E2Y@̇DY@w^9hZ@jZ@zyJZ@c~Z@l jDZ@?w3Y@pٓ1Z@GZ@{QY@x"ֹ-Z@09pY@=?Y@S(Z@IpZ@T$DZ@$Z@*ҦJZ@ˠqZ@qȗZ@`-Z@jlZ@8X@YUY@Ps Y@f Y@FX@5JOX@w6Z@ dZZ@"}Y@23Y@ Y@tiy~NY@ctY@QMZY@N](/X@$%X@RX@Yc"i;X@ffOW@oL W@)"N=W@5k)W@&̂p[W@_W@h>W@`YvW@mHv>W@qGW@*wW@DrcW@K$W@TZwX@gW@$v%W@<UٌX@\[cFY@\[לX@bX@܇ X@cTZXPX@QzX@G۳ cX@B gX@+ X@PZX@bӢ,]X@g`X@ON\vX@SΝX@όRϪX@]6wW@O}W@&GW@Y9DCW@fOtW@UM8X@}[X@5J$X@X:#X@^W@X< W@CD%W@ʼnxV@DGcU@ֆV@$V@zxZOU@CѷT@~S$T@IT@V(U@9,mOU@Mdd~U@MU@T[[U@q"7ZU@RVU@nPJU@=w<T@3NS@BbGS@KKS@;O~R@ "~rS@`a-S@>] 2S@;lS@/S@e)S@?\S@;}S@xĥS@ZJS@)˨qT@w wtT@bSwT@ޯT@T@¾S@K`U=T@V 5T@ՁT@ iJT@׸uJXT@^gȌaT@T@鄵hT@ gT@A$IT@S@aWS@^#bS@\h]ÐS@811S@-OT@8}kT@'|:T@<2GT@Z!WS@S8S@S@X[/T@BbT@cuT@L~T@̽#%U@O(SlU@fmdlmU@ѪVyU@_k+]FU@<T@KL~$T@Dh!#T@.T@2BYQU@.U@^<'U@aV@x򱌧V@8g`V@aGoV@Ŵ]W@2wW@nπNW@d`W@,a[5W@/<7V@Z0hV@JV@V@-bFW@?V'W@sg,W@+jCRV@K+}V@_xV@%TV@zU@R1PuU@vU@v\U@%U@*~OV@[l^V@_,V@7{@V@BU@`U@1U@hI<+U@YwV@zϿnU@ۖ U@7|-V@l`W@ W=#V@K@V@xV@W6V@~)gjV@I ZV@cf FV@NV@EӐV@}hV@l=fW@"KV@ǬW@rD W@ ~V@X6a V@*6V@{H)V@bmTV@Ḃ{V@o¢4V@XV@ζV@G@}U{GV@>qV@$ǙdV@hOU@r;U@<KU@=+UAU@倄&T@Z(T@u]T@Lj#UT@(œT@ v0T@=7T@OT@T@*T@aWT@|{;T@{RT@*ΜFU@\/%%U@i5Q"U@99,U@Ug V@0U@V@EU@WleU@-";U@ F_SU@/pU@1q0U@-VεU@X U@O{fV@/!U@ދyU@PSU@( {S@AQS@3DS@F+R@bĢ~S@ S@S@ T@S@d^S@D S@3Vh2TS@L 0S@* |S@eWS@P\ߜS@ʱS@Ny0S@2uT@\䂮S@~JS@(S@0t NFS@wR@NYfS@ӲS@aq=S@/ÊؗS@?hnS@vqS@'NS@m$j:S@K/dS@r`S@;5Q@n5Q@N+Q@ #ܦQ@Oy&"yQ@13wQ@|*+R@[WR@ oMXR@jo"YR@]?sR@2r22S@nS@H7 S@(.R@pކR@JFm9R@yR@:&%,R@QSR@ 7R@ʍR@8"W#R@¹.R@_,\R@[R@}=JQ@UxQ@2`Q@#uؚQ@,vf-R@nR@DL%2R@$,R@ֿ Q@\Q@8\R@ Q@7P2/R@jER@2*T_R@gBO`R@0R@ܺR@{/R@.-1R@ ]R@;R@h]ne/S@NR@@S@cR@I)#R@Ub+R@HbVR@kWQUR@k R@Ӻ#R@h٬R@MK$R@mlR@#Z3}R@%¦R@ Q@.|Q@$]Q@Q@VQ@:rQ@eSR@w )R@lQ@+Q@@ Q@!Y*R@M^f~Q@i[@+T[@~opQ[@ẽO4[@Lu[@JXY[@͹w2[@$`3[@O[@WTp[@LE'0[@Qc[@dea4h[@I/[@՞}1[@7@(H5[@ DATA-END DATA-BEGIN DikP@iaxP@sQ/P@Զ8P@^[P@]/]YsmP@5# #%P@N-yO@1LSTO@OqO@0 aP@*vdP@ǵ `Q@9`*GQ@0( Q@$Q@E*;R@(cR@հx[R@ KzR@WBR@<ۗt\R@m8CnR@;zR@ˇR@xeR@ٺtoR@ )R@JQ@FSR@EUGQ@7Q@iQ@ۂQ@RƪCR@z.R@W@>vGR@jU0R@uGR@]).R@CR@f)R@=R@?PhR@SkR@(1CR@QR*HR@-tAXE@ ND@;TxD@3sD@ _D@ h~|D@h:C@`\ B@iB@JZB@2C@D:dC@ƃB@ C@JA@aylA@@@;A@@@n@@8N6#@@|12?@lA3$@@p?@/ @@dGA@ym}_@@=R&@@9WG@@.F)@@џ@@>J0,@@r(PK @@JK @@E-,@@V@@3)@@+r@@{&@@Rk_@@mdZ{ @@1=?@B!@@x"?@72@@%?@xH@@FG&?@L(\?@vǭ?@YL?@;ppL?@Uh?@s?@> p?@\DS?@T?@ u1?@r=a?@s., ?@4j~:?@e>@i/?@ N@GS>@z @@_n">@)v>@`K^>@}9>@e_?@7+|9?@w?@0?@*?@_,u?@q>6D@@|oi?@=@?@a6C+?@/qTż>@Zt?@[ b?@At>@m?R?@=Ö>@Hu/E?@Z&@@ȉ @@޾\?@?@U2n?@5@7=_ O@SN@ӋnN@b2UwN@ȸݹ-N@dM@@ԱM@veYM@%5M@eŅL@EL@;M@%8L@n#L@K@#IuG@;7WK@*BEK@arpG@̮݂QG@YKF@*JE@dlF@,rX[F@{cF@ F@ b\ E@/ @,@@UTyR@lwkR@phR@&iPQ@^OUR@ tqHR@5-(R@EuNeQ@CQ@!F?Q@nQ@XQ@YCQ@obQ@@Q@ۚEP@w8N@69fVN@MN@~eΛN@$^O@eFO@("O@dIO@Zq2P@jtP@&HTP@ݵP@gP@>츠P@OvPP@CL@_DJ@q<=K@$KK@HK@ :x}J@^WZJ@4hJ@O]J@0IG@yACF@6/){F@vF@VF@R2TG@"H@.9G@d-QI@ VJ@s=I@NVI@QfjOH@VfpH@yJ I@ H@7}7C@BrB@md"B@)MB@v@΄B@  >@K@?@~ ?@I%/(u?@y/n?@+\ ?@o?@NdD?@\p&@@58v?@/v(w>@e>(?@xk>@Śr@>@Ee3b?@t?@+;[@@/`,A@o˭A@@b)ٯA@D 3B@*B@nhA@1 @@tƭ{@@$@@)3EakC@3bC@0C@գduE@裠E@ĶE@#iZ>E@ą}o:D@iBBoD@|RD@aqLD@0 \M@%M@#k"ȢNM@sZLM@$[L@0ʋL@]FL@nP@BMUuP@BaP@jM~R>P@WX# *P@ŭBaO@9pgO@_AP@,aO@cYgP@" #P@F/P@ lx)Q@BcQ@vKQ@KKQ@|z<Q@T0R@FM2 R@x:R@;PϣQR@>t(`R@nBR@pSOR@جw0R@0Q@hQ@[RᗽQ@VwQ@Q@iV} R@{zQ@;DߟgR@?ኺQ@lz R@eQ@QCb_R@<Q@${"KR@#t7 NR@]t"R@6ϵR@6SyR@/<*R@!*(>(R@ qD@vע/D@II!XiD@9CD@3X?QC@Ay B@xكC@wSHC@aĩnC@U+%B@XhdrB@"dB@x̄A@z*@@[>fQ@@5Iw@@@@ϭ+N}@@߸@@yA@4PPA@"싫'uA@;R>@@Y^@@5 y @@Ɉ@@=ҴP@@8&"Q@@g豧@@Iĭ @@mt@@A@@j8@@=b}@@FE/@@11+w@@4M;@@n@@zX@@9 & e@@u@@KY@@O-@@y1`@@!ב@@Qp9@@/c2@@ĎA:@@*3@@Z*@@@]^@@$/6@@hM?@"O@ָeJ@6J@BBüaK@辶y3K@J@sJK@[K@۩"L@& }G@q}E@bsxF@d0eF@7J F@(;F@侌HsG@!>J\G@0bE@9cE@;Q E@L2K@@EbMK@@ԦA@@"KR@woD R@ Pi5Q@ KQ@Gn*R@ؕQ@˜FQ@&]O'gQ@gc=Q@^C#Q@ތ%Q@?P@:ZU{Q@tP9P@̳`\O@f4P@UzdN@tM@QNfY N@oˑN@6m;O@3lO@-s8ΦN@SӨ(O@}"ZP@wIP@D?P@\Nh{P@WP@2ykSL@K@MSL@rG5J@ w{{J@J@K@6L?RKK@S3J@VRPK@0{,J@Ԉg(G@`V*G@YRG@lAF@ѣ?F@uzKF@?F@ AG@{AضI@՞I@%I@wk`I@8iH@0sM $H@mW#YH@H@IΊC@kt* C@Yץ0?C@zTpgB@YB@7K ?B@m0^B@YOM?@ /6@@:jͭ@@9#{@@@Ι3@@#x@@)?@g?@ZEA@QKA B@ʢA@`)[A@a@ $A@-w@@ Dx:@@m;2@@),@@@|E@=IE@UjFE@QD D@cyW D@$?CXBD@^#LD@葽M@1L@R3$M@c?\M@jLTL@`:-O@K kO@)uO@V٧P@=d,P@\q#BP@x/eP@lozP@6cQ@ئ)'>MQ@[%Q@D*Q@<: P@CNwP@7P@tWP@zNQ@e:GR@r R@4ص!R@7}ZR@!RQ@2/Q@aBQ@ E>!Q@DMLQ@d4rQ@>5Q@:=Q@f^Q@'Q@XQ@%qnQ@4*F R@_Q@R4Q@-ɇ6Q@vQ@㻑Q@eQ@Q@XATQ@OCQ@UQ@;Q@^4gQ@FQ@vM%Q@CnQ@LvC@p3B@9qfC@BNB@ yB@lH4B@L&GA@?I%A@ܥA@ȮPJ|A@>J@@3^uGA@ A@|mrA@&(A@+A@A@~$(A@SaA@,A@DA@rA@%tL@@>V%A@/7@@A@*8@@ @@ ?@@l*@@@@uhA@P@@:!e@@\?$@@s!:)Ʃ@@А@@ Z l@@*gs@@_7K@@M@@Y@tc@@,7Ij@@݄O@@jv&@@Dd@@z!T@@Kvh@@H<@@iq@@ aġi&A@MBZA@,7@@ 0x A@yU@@O @@<Š@@D @@m&d@@a\@@@@h4@@x>L@@+A@qY A@N {@@7{@@:,X@@ eYa@@n`@@xt@@k:@@x`Iޫ{@@! *}@@B@@ s+"O@cN@S0N@ dG@+cPH@ԼZH@}3H@4jH@4aZGI@[$sJVI@I@sI@nfO3H@HJ C@ïB@MC@r[A@>}~A@-bNA@QA@ 2/A@}гAA@\{`A@_NA@%k,AA@4uoA@AX A@A#@@$Y@@vL'/A@l),A@TySaA@\A@c;A@3/2A@OƇ24A@35;A@J@@K$!@@ӰA@I;@@~bz,A@@@&O@'!KG@{KlH@4aH@.U H@P;FH@LkK@3X@K@l>J@J@ tJ@\ {I@hƿI@k~0NJ@OI@VÅQmZI@p IYI@-pK@uK@_0L@=9XL@ڽG2B@9boB@~e B@ݔ@@ׄTA@azG:A@,5;A@ @A@ae%\A@/VaA@),A@4N'/A@Z~(Z@@䖢@@$_D@)3cNbE@N~&E@9ҥD@aiD@Mo#M@yo&BP@-ceeP@I}P@wg*P@:QQ@>J?Q@Z*bZQ@&>5GQ@:бQ@g$Q@vmP@)·DqP@;P@dͧQ@ ǸfQ@uQ@bďQ@]Z*Q@IX!jQ@},pQ@@$_Q@YQ@#gA9yQ@p?Q@DnQ@cP@>5Q@M*P@-O@poO@7}O@\~GP@B(eP@ŌuԓP@^P@ryFP@KP@x1DQ@NMP@ɲC`P@"?!?P@P@RQFO@Tq.O@<=ywO@-b95O@ޞAN@MON@SonN@KU,N@ `M@z^=M@+$$fM@Q L@)=L@fxG]L@7:hL@-K@h(K@(TK@K@TZJ@J@9k@LJ@2z_u J@wKI@n.B@0\#B@|_KVLA@x9B@2']A@R gB@cvA@`%<A@oGA@M"B@!)A@:CvgA@ӖA@?TA@%SA@Z B@ %TA@b8O A@o,̝A@RN{A@lPA@J`A@K3vA@߀޽A@W_S+B@MIB@A!A@0˾0B@(ϏB@ (*`A@U2tA@G3HA@rK\A@=hpA@*'%A@B@A@> B@hpB@K7dA@oEA@=7&A@s]xidA@JPA@g;ôA@>A@`-A@H y[A@b CۯA@vBp#M@#&O@ #N@&% N@w!5rN@*)6N@8G@W˼#^H@'r eH@kK@=J惘I@fB wJ@nذOJ@mXx17I@}L1eJ@PJ@ PGAK@tPI@EH@(畵I@bW:[I@zDƱH@RDK@.xS$LM@ܩL@.vIL@ +s!M@g\L@ƒ2L@ۘ YK@D$0M@NUM@L4 N@1P}˒G@nG@[G@8D@xdD@҅˙}D@tWnG#D@,JD@`@7E@CexE@+E@EE@pF@ܠuF@6IF@ZF@O'B@+`&NC@ C@.lp%C@ιTC@EtB@ B@7'B@!'oB@RjB@{ 8*B@7<B@=;pB@Vu8*B@; +B@vdPiKI@MI@&;I@+|H@nvG@.MzbG@8 Y8H@fyYG@qF@tz nE@lG:E@$-F@`&E@߯F@ }z2G@4fZF@ߍH@?I@T_@H@&5BlD@0`illC@on)C@Υ.[(D@uQ!C@]>|LnB@SyY<9A@VFuA@[nsA@KThgA@7*B@mQAp'B@A@_HA@`jA@n%EB@Vt(C@MURB@TD@F%*E@wYWD@PeUzP@"%P@Wz4lP@\P@8ЀQ@(Y Q@TȝP@ }eWQ@fX˚cQ@xb;EQ@ w|UOQ@R0Q@sTR8Q@,SQ@1$LQ@d>Q@F݇P@udP@NR:wP@:P@GP@sqP@`{P@X9`P@P@}P@ǧP@욮oP@+xP@aP@KؖX>P@E(%P@ 6kO@XwO@hP@|a\O@uO@~bO@c@ȇO@! kTO@=ywO@W O@UpO@QW?O@m.#P@)$UP@V%GP@s׿bP@H؄RP@JP@%0/P@5^FP@S]8P@nvmP@i[P@Q$.\uP@@SV9P@?JP@183P@yzO@JE(h]O@@(ZBO@O"ZbO@16+O@Z8IO@h$1O@nYRO@ZO@$ rO@_IO@uj+O@N@6kLB O@ן8]eO@)}'PO@fKO@q7kEO@\o P@,b*P@^(i P@=P@H#tO@NO@شyO@Q-C"P@dl4P@a%)P@͐:FP@ 2P@۞J ZP@,TP@ʦ~tP@^4>P@2WP@0P@%.{P@P@kjP@8P@{ԤP@ܛP@^tP@BץP@hBuP@NiP@;IuP@KWP@ơvpaP@8 P@ԃP@ǞeP@tmP@qtBP@wZKP@{:,P@ؚ͞P@\ǟP@ya.P@T*2gP@nXP@M*Q@ Q@k*y#Q@*½Q@`9 P@G bP@vHP@JcQ@IR5Q@8'Q@^P@ܳY:P@EۙNP@`@P@:jO@yߨO@}KP@Ne=P@ !_O@"O@<O@B_ivO@< uRO@Ш=W)O@/yrN@#G@z G@:7G@d6"|G@ilG@cG@dFG@kvF@?(F@459F@*%F@ɴt+G@|9G@fuTG@P'݁G@c lG@iqG@VH@hIG@PG@F̮G@vG@!ܒG@2H@=wt%H@iH\H@J%H@QؔrH@a]jlH@GH@xhr%H@㶛 I@!H@fb?I@O;I@:+H@9WS(H@JǨH@pm-I@#ͨyI@>|sI@}SI@HeţI@{VI@ R`I@N;~%I@OI@QH@H@X+}H@\RH@H@u~kH@LD; I@tH@qxOH@&~^ YH@HH@aOD[H@1?`?:I@156,I@ dI@8(9I@I@ȍGI@VJJ@@J@*"I@~_I@I@I@7BJ@lȵ3J@lJ@i]H@d¯|H@龰z~H@^}>=H@eV2f4H@A|G@""G@: G@y[G@s.;G@:|VG@Ѥ8H@D'ዺG@'SjvG@WoAxG@4Ϡ7G@ G@PHG@QG@lNG@$WcF@7+OF@k9sF@WiץpF@p{F@hAF@rF@ZF@^(F@JyL#CF@3٠^KF@OmF@/yF@OxfF@CaF@$)F@\֯F@]EC:F@m䜻F@)r2G@W.G@pF@gBF@!OF@RG@GakG@IzfG@B"EG@MOF@ jqF@]-6?sF@Bܠ43F@*F@ E@%ԙ.E@@UHE@R'E@kJ`EE@+E@BD-F@%{E@ e+nE@LǪoE@u*-0E@:#G@-C?F@AF@]NF@AF@o$TE@)B@'WC@A7=C@WfFgC@3LC@W!B@VăAB@og@B@>i1zx^B@jA3mB@RB@(zB@h_CB@a5}B@KHiB@/B@x"meB@\OuC@ 28rYC@s7C@kTdC@.5B@ C@HB@[3C@pC@ٱnC@O C@C@pqAB@JMxB@ˌB@/#B@}f 'B@?$B@Ck:B@)yv\A@YVB@e_cIB@CXwB@;iB@%0ҮB@30B@鬵B@aRB@ݹIU:B@N*LB@y_96A@{A@2F\_A@@,뵼A@ƍ{~B@z@(gB@.1B@:NFB@c_sB@|B@yG kB@lxB@G~SB@XB@< @C@{B@RITC@FHB@cNC@e!C@/\9C@f7kC@){C@f0 C@ +QC@`X9C@:"C@ ݔC@RbC@#-C@ZOD@سU D@Rj?D@aD@9lu_D@@}(ZD@UD@gqD@yTgD@ ~hD@6*E@3"&E@`D@):D@ǥD@ E@l yE@"lFE@;E@Q|D@2XD@XpD@u$fD@FoD@ävh*D@6z"D@N)D@ VD@qM:D@.BD@7 D@[D@p(d vE@L#E@KxF3FE@,tE@1E@ޖE@g'EEE@x'ھE@P6E@EC_E@UR}kE@{QE@@#F@ڵv F@ gMF@ ۳?F@6cE@GFPE@@hzE@"E@FP#6E@e9b-E@0lD@}E E@g-`wlE@ڥ]E@a4VE@Ə1>9E@OeD@})7D@eܬD@DĵD@,T`LD@p`D@c>1D@dSC@DtC@!%*fC@3RX]C@&qC@QrۗC@KqpC@|BC@mU[SC@q"`DC@5VC@v(C@H7D@>iZxD@l9ED@B$D@C@ gCD@8LRD@2LFD@ mLD@N2D@څ{C@ԡ>C@k+C@5qC@u C@\.C@iذ5B@8B@9]C@tPC@SC@,C@B@fB@~edB@lC@ ]D@}pC@ҵO&D@{T{D@W-D@CJm,D@d D@nD@w\}D@Wի"D@;?VfD@nD@aœE@,:E@ s{h/E@⡁lLE@/ޱE@VE@ΕE@IiF@F7tF@a0dF@Gbm|F@^yF@z:r#F@J_MD6F@F@ʂ_`UF@Y@F@zF@ F@ -F@\\F@zoQrF@d&AF@]VOE@9E@3.E@l+nE@wE@1 E@L! E@GF@2]E@`G@QD@@)>lD@K3smD@6.D@#E,,%D@뻞C@|UBC@֡C@L>C@/d٠C@mC@!"mR(D@ZMC@x)jlC@mC@c/C@h巊C@˴g7C@. ?C@/)e C@h-6B@o"{TB@ůُB@Oef^B@xܘgB@7B@{̓BQB@ _QB@nVӉB@C`u5B@Qrahk(C@u&^%C@GB@ aB@J:B@)5C@BO^C@vg²YC@aC@+,B@2snB@"`QpB@c &3B@'B@j@tA@r,A@%*B@-MŵA@{wѶ&B@5i\A@\A@x B@B@,fC2B@QG'aM@t M@|q#L@f8+M@M@M@NNM@@mM@L@yAM@ HA\M@rYRL@]y6^L@nȋL@-FL@48[N@Lg4N@$iM@X4KM@{%`N@7 !N@`LN@})գN@M@O.M@tFM@+'$N@ OYM@cqׁM@oIUGM@W|&L@pL@ŪL@gJ" L@H0M@q\CmL@c@BK@Bh$K@n9ZK@f>K@_<[LK@fJ@X[K9K@;6-bK@qCqK@ګ>K@3&K@F@p:K@RL@.SK@%=eL@AlgL@Nu]L@!yL@ƅޤL@Ks@L@́b8L@dCL@88L@ zL@DOEL@((L@@5h_L@(hL@:5K@yUK@%_K@c>mwK@sFM@p+M@< 'bM@J$M@11 M@-L@-#M@3lM@mHM@"ΟM@LM@$qEM@20N@$M@{h(N@CpN@{sJ@6hOJ@;NJ@6{3J@ ZeJ@*)v$J@ғ-J@c}rJ@EJ=J@WHJ@kTJ@D)ԡJ@zcI@ ʖI@2(mN.I@4elI@m?K@ &K@-[J@ ~oʤJ@GAi9K@|*JK@*I@&h"YI@׌5/gI@b9I@ێZH@PvnrI@'LzI@CN^iOI@gH@pvH@HoH@RH@7?EI@ȺFtsH@|k}H@%QH@dG@I@O7kI@N(0 I@qMBJ@I@_f̪I@y8I@a *J@ԀYJ@jeJ@7J@EI@J@o`DJ@rJ@]eB(K@!~|LWK@a(aK@E!?3K@ƆJ@U*M^gK@Fv[@K@tEbJ@IOӍJ@H^J@M+_tJ@ ;K@nJ@&W,tJ@CzQIJ@f]I@xI@BMI@ʈυ4NJ@:#R:wJ@J@MI@p!nZI@?1[I@lnH@5hc H@ SfH@%ӟH@S[K@L[K@d0L@}%K@b<. K@K@颳"L@ m{QL@yXL@#+L@.lK@B|EL@L@D L@}XI&M@3CMHM@Ϋ5M@ΙӸL@7L@Bi\L@cwx,M@{(QG@{bG@gk5G@X,:9F@nqG@F}G@&QG@ċF@ؕ F@e'F@xHF@{UDG@7mF@ cezF@7wrMOF@=|CAE@w|G@V/]G@?ndG@F^G@Ё̒G@ v36G@ vWnG@G@)2`F@F@vaG@ &@jG@7:E1F@X F@+ XrF@۔F@-9NE@,E@.JF@ȆlF@ÇE@O56E@>NF@XW[F@pF@dDK}F@C2eF@E;z F@AF@/)F@k\(vG@a\G@E@lpE@Ր֠JcE@O: E@!wE@}E@݁sE@T(E@;52D@GuWD@r'HE@UVrrlE@D@clD@:rD@=P9D@'Ę5G@D@?@&D@mVx)uD@wDD@Pji}D@ẐD@۞;D@sP&D@>wD@\C@Oq3 C@$D@ JxD@޽fhD@-ޥGD@}D@I#D@)CZD@=D@lޗE@ⱧxE@QE@u 0E@2{E@&ˆ"E@OL/E@/ O{E@-ޖ"E@К&D@UaD@ٓ#E@ |E@5D@f25D@LiiD@Tf*'D@)$E@PMʤE@˸E@F@l?uF@,BF@vF@$},F@nF@G@ɇYGoG@; EC@*)C@ړ|C@"q!C@KPC@ȤC@>C@NB(C@XNC@zC@%C@qWC@ZKLB@bJjB@sB@M:C@8lN6C@2C(C@qFLC@l ~iB@"넆B@$B@~EB@8B@e`\B@C@C?C@_p)C@ C@DI@soH@rH=H@?|~H@RG@ 7G@ExG@չ=G@k5OF@/:43F@uZCsF@D n{E@me-0E@2*goE@I&IE@?˙F@>II@$RD@D.D@1mD@KC@`/C@;!mC@ǤC@íB@뿹3B@SpB@8ŵA@'R[A@tA@B@q*L.D@ DATA-END ATLAS-IO-END eckit-2.0.7/tests/geo/grid_reorder.cc0000664000175000017500000000554415161702250017704 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/Grid.h" #include "eckit/geo/order/HEALPix.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" #define EXPECT_EQUAL_VECTOR(x, y) \ { \ EXPECT_EQUAL(x.size(), y.size()); \ for (size_t i = 0; i < x.size(); ++i) { \ EXPECT_EQUAL(x[i], y[i]); \ } \ } namespace eckit::geo::test { CASE("HEALPix") { SECTION("HEALPix::reorder") { std::unique_ptr spec(new spec::Custom({{"grid", "H2"}})); std::unique_ptr ring(GridFactory::build(*spec)); static const Grid::renumber_type expected_ren_ring_to_nested{ 3, 7, 11, 15, 2, 1, 6, 5, 10, 9, 14, 13, 19, 0, 23, 4, 27, 8, 31, 12, 17, 22, 21, 26, 25, 30, 29, 18, 16, 35, 20, 39, 24, 43, 28, 47, 34, 33, 38, 37, 42, 41, 46, 45, 32, 36, 40, 44, }; const Grid::renumber_type expected_ren_nested_to_ring{ 13, 5, 4, 0, 15, 7, 6, 1, 17, 9, 8, 2, 19, 11, 10, 3, 28, 20, 27, 12, 30, 22, 21, 14, 32, 24, 23, 16, 34, 26, 25, 18, 44, 37, 36, 29, 45, 39, 38, 31, 46, 41, 40, 33, 47, 43, 42, 35, }; const Grid::renumber_type expected_ren_none{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, }; auto order_ring = ring->order(); EXPECT_EQUAL(order_ring, order::HEALPix::RING); auto ren_to_ring = ring->reorder(order::HEALPix::RING); EXPECT_EQUAL_VECTOR(ren_to_ring, expected_ren_none); auto ren_to_nested = ring->reorder(order::HEALPix::NESTED); EXPECT_EQUAL_VECTOR(ren_to_nested, expected_ren_ring_to_nested); std::unique_ptr nested(ring->make_grid_reordered(order::HEALPix::NESTED)); auto order_nested = nested->order(); EXPECT_EQUAL(order_nested, order::HEALPix::NESTED); ren_to_nested = nested->reorder(order::HEALPix::NESTED); EXPECT_EQUAL_VECTOR(ren_to_nested, expected_ren_none); ren_to_ring = nested->reorder(order::HEALPix::RING); EXPECT_EQUAL_VECTOR(ren_to_ring, expected_ren_nested_to_ring); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/iterator.cc0000664000175000017500000000445715161702250017070 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/geo/Grid.h" #include "eckit/geo/Iterator.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("iterator") { auto test = [](const std::string& gridspec, size_t expected_size) { std::unique_ptr grid(GridFactory::make_from_string(gridspec)); ASSERT(grid); EXPECT(grid->size() == expected_size); // test Grid::to_points auto points = grid->to_points(); EXPECT(points.size() == expected_size); size_t i = 0; for (auto iter = grid->begin(), end = grid->end(); iter; ++iter) { EXPECT(points_equal(*iter, points[i++])); } EXPECT(i == expected_size); // test point in Grid size_t j = 0; for (const auto& point : *grid) { EXPECT(points_equal(point, points[j++])); } EXPECT(j == expected_size); // test NextIterator decltype(points) next_points; Point point = PointLonLat{}; for (auto point : *grid) { next_points.emplace_back(point); } EXPECT(next_points.size() == expected_size); size_t k = 0; for (const auto& next_point : next_points) { EXPECT(points_equal(next_point, points[k++])); } }; SECTION("HEALPix") { for (size_t Nside = 1; Nside <= 3; ++Nside) { test("{grid: h" + std::to_string(Nside) + "}", 12 * Nside * Nside); } } SECTION("Reduced Gaussian") { for (size_t N = 1; N <= 3; ++N) { test("{grid: O" + std::to_string(N) + "}", 4 * N * N + 36 * N); } } SECTION("Regular Gaussian") { for (size_t N = 1; N <= 3; ++N) { test("{grid: F" + std::to_string(N) + "}", 8 * N * N); } } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/test.zip0000664000175000017500000000064415161702250016425 0ustar alastairalastairPK hZaUT VgVgux PK hZb/UT ]g`gux PK  hZBUb/cUT dgggux d PK hZaUTVgux PK hZA;b/UT]gux PK  hZBUwb/cUTdgux PKeckit-2.0.7/tests/geo/kdtree.cc0000664000175000017500000002372515161702250016514 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/container/KDTree.h" #include "eckit/geo/PointXY.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { //---------------------------------------------------------------------------------------------------------------------- struct TestTreeTrait { using Point = geo::PointXY; using Payload = double; }; //---------------------------------------------------------------------------------------------------------------------- /// \brief Class used to test whether any point in a kd-tree lies in the interior of an /// axis-aligned box. template struct PointInBoxInteriorFinder { using KDTree = KDTreeX; using Point = typename KDTree::Point; using Alloc = typename KDTree::Alloc; using Node = typename KDTree::Node; /// \brief Returns true if any point in \p tree lies in the interior of the specified /// axis-aligned box. /// /// \param tree /// Tree to search. /// \param lbound /// Lower-left corner of the axis-aligned box. /// \param ubound /// Upper-right corner of the axis-aligned box. static bool isAnyPointInBoxInterior(const KDTree& tree, const Point& lbound, const Point& ubound) { if (!tree.root_) { return false; } auto& alloc = tree.alloc_; auto* root = alloc.convert(tree.root_, static_cast(nullptr)); ASSERT(root != nullptr); return isAnyPointInBoxInterior(root, alloc, lbound, ubound); } private: /// \brief Returns true if the point stored in \p node or any of its descendants lies in the /// interior of the axis-aligned box with bottom-left and top-right corners at /// \p lbound and \p ubound. static bool isAnyPointInBoxInterior(const Node* node, Alloc& alloc, const Point& lbound, const Point& ubound) { if (node == nullptr) { return false; } const auto& point = node->value().point(); if (isPointInBoxInterior(point, lbound, ubound)) { return true; } const size_t axis = node->axis(); return (lbound.x(axis) < point.x(axis) && isAnyPointInBoxInterior(node->left(alloc), alloc, lbound, ubound)) || (ubound.x(axis) > point.x(axis) && isAnyPointInBoxInterior(node->right(alloc), alloc, lbound, ubound)); } /// \brief Returns true if \p point is in the interior of the axis-aligned box /// with bottom-left and top-right corners at \p lbound and \p ubound. static bool isPointInBoxInterior(const Point& point, const Point& lbound, const Point& ubound) { for (size_t d = 0; d < Point::DIMS; ++d) { if (point.x(d) <= lbound.x(d) || point.x(d) >= ubound.x(d)) { return false; } } return true; } }; //---------------------------------------------------------------------------------------------------------------------- /// \brief Returns true if any point in \p tree is in the interior of the axis-aligned box /// with bottom-left and top-right corners at \p lbound and \p ubound. template bool isAnyPointInBoxInterior(const KDTreeX& tree, const typename KDTreeX::Point& lbound, const typename KDTreeX::Point& ubound) { return PointInBoxInteriorFinder::isAnyPointInBoxInterior(tree, lbound, ubound); } //---------------------------------------------------------------------------------------------------------------------- #define EXPECT_POINT_EQUAL(a, b) \ for (size_t i = 0; i < Point::dimensions(); ++i) { \ EXPECT(a.x(i) == b.x(i)); \ } CASE("test_eckit_container_kdtree_constructor") { using Tree = KDTreeMemory; using Point = Tree::PointType; // build k-d tree (offline) Tree kd; std::vector points; for (size_t i = 0; i < 10; ++i) { for (size_t j = 0; j < 10; ++j) { points.emplace_back(Point{static_cast(i), static_cast(j)}, 99.9); } } kd.build(points.begin(), points.end()); // size EXPECT_EQUAL(kd.size(), points.size()); // pick a point auto ref = points[points.size() / 2].point(); SECTION("test single closest point") { // a point similar to an existing one EXPECT_POINT_EQUAL(ref, kd.nearestNeighbour(ref + Point{0.1, 0.1}).point()); // exact match to a point EXPECT_POINT_EQUAL(ref, kd.nearestNeighbour(ref).point()); // off the scale, i.e. not within a group of points (+) EXPECT_POINT_EQUAL(points.back().point(), kd.nearestNeighbour(points.back().point() + Point{1000., 0.}).point()); // off the scale, i.e. not within a group of points (-) EXPECT_POINT_EQUAL(points.front().point(), kd.nearestNeighbour(points.front().point() + Point{-1000., 0.}).point()); } SECTION("test N nearest") { // move this point so it lies between four equally, make sure we differ by 0.5 along each axis auto test = ref + Point{0.5, 0.5}; for (auto& near : kd.kNearestNeighbours(test, 4)) { auto diff = near.point() - test; for (size_t i = 0; i < Point::dimensions(); ++i) { EXPECT(Point::distance(Point{0., 0.}, diff, i) == 0.5); } } } SECTION("test a custom visitor") { // Test a custom visitor. The purpose of doing that in this test is to ensure that the public // interface of KDTree is sufficient to write a custom class traversing the tree. auto a = Point{0.25, 0.25}; auto lbound = ref - a; auto ubound = ref + a; EXPECT(isAnyPointInBoxInterior(kd, lbound, ubound)); auto b = Point{0.5, 0.5}; lbound = lbound + b; ubound = ubound + b; EXPECT_NOT(isAnyPointInBoxInterior(kd, lbound, ubound)); } } CASE("test_eckit_container_kdtree_insert") { using Tree = KDTreeMemory; using Point = Tree::PointType; // build k-d tree (online) Tree kd; std::vector points; for (size_t i = 0; i < 10; ++i) { for (size_t j = 0; j < 10; ++j) { points.emplace_back(Point{static_cast(i), static_cast(j)}, 99.9); kd.insert(points.back()); } } // size EXPECT_EQUAL(kd.size(), points.size()); // pick a point auto ref = points[points.size() / 2].point(); SECTION("test single closest point") { // a point similar to an existing one EXPECT_POINT_EQUAL(ref, kd.nearestNeighbour(ref + Point{0.1, 0.1}).point()); // exact match to a point EXPECT_POINT_EQUAL(ref, kd.nearestNeighbour(ref).point()); // off the scale, i.e. not within a group of points (+) EXPECT_POINT_EQUAL(points.back().point(), kd.nearestNeighbour(points.back().point() + Point{1000., 0.}).point()); // off the scale, i.e. not within a group of points (-) EXPECT_POINT_EQUAL(points.front().point(), kd.nearestNeighbour(points.front().point() + Point{-1000., 0.}).point()); } SECTION("test N nearest") { // move this point so it lies between four equally, make sure we differ by 0.5 along each axis auto test = ref + Point{0.5, 0.5}; for (auto& near : kd.kNearestNeighbours(test, 4)) { auto diff = near.point() - test; for (size_t i = 0; i < Point::dimensions(); ++i) { EXPECT(Point::distance(Point{0., 0.}, diff, i) == 0.5); } } } } CASE("test_kdtree_mapped") { using Tree = KDTreeMapped; using Point = Tree::PointType; std::vector points; for (size_t i = 0; i < 10; ++i) { for (size_t j = 0; j < 10; ++j) { points.emplace_back(Point{static_cast(i), static_cast(j)}, 99.9); } } // pick a point auto ref = points[points.size() / 2].point(); auto passTest = [&](Tree& kd, const Point& p) -> bool { // perturb it a little // we should find the same point auto nr = kd.nearestNeighbour(p + Point{0.1, 0.1}).point(); for (size_t i = 0; i < Point::dimensions(); ++i) { if (nr.x(i) != p.x(i)) { return false; } } return true; }; PathName path("test_kdtree_mapped.kdtree"); // Write file with k-d tree { if (path.exists()) { path.unlink(); } Tree kd(path, points.size(), 0); EXPECT(kd.empty()); kd.build(points); EXPECT_EQUAL(kd.size(), points.size()); EXPECT(passTest(kd, ref)); } // Load file with k-d tree { Tree kd(path, 0, 0); // Cannot insert point as the tree is readonly EXPECT_THROWS_AS(kd.insert(points.front()), AssertionFailed); // Cannot build with points as the tree is readonly EXPECT_THROWS_AS(kd.build(points), AssertionFailed); EXPECT_EQUAL(kd.size(), points.size()); EXPECT(passTest(kd, ref)); } } CASE("test_kdtree_iterate_empty") { using Tree = KDTreeMemory; size_t count = 0; Tree kd; for (auto& item : kd) { count++; } EXPECT_EQUAL(count, 0); EXPECT(kd.empty()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/grid_healpix.cc0000664000175000017500000001542215161702250017670 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to itr by virtue of its status as an intergovernmental organisation nor * does itr submit to any jurisdiction. */ #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/grid/reduced/HEALPix.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("gridspec") { for (size_t n : {1, 2}) { for (const std::string& suffix : {"", "n", "_nested", "r", "_ring"}) { const auto spec = "{grid: H" + std::to_string(n) + suffix + "}"; std::unique_ptr grid(GridFactory::make_from_string(spec)); EXPECT(grid->size() == 12 * n * n); } } spec::Custom spec1({{"grid", "H3"}}); std::unique_ptr grid1(GridFactory::build(spec1)); EXPECT_EQUAL(grid1->size(), 108); EXPECT_EQUAL(grid1->spec_str(), R"({"grid":"H3"})"); spec::Custom spec2({{"grid", "h2"}, {"order", "nested"}}); std::unique_ptr grid2(GridFactory::build(spec2)); EXPECT_EQUAL(grid2->size(), 48); EXPECT_EQUAL(grid2->spec_str(), R"({"grid":"H2","order":"nested"})"); spec::Custom spec3({{"grid", "h2"}, {"order", "ring"}}); std::unique_ptr grid3(GridFactory::build(spec3)); EXPECT_EQUAL(grid3->size(), 48); EXPECT_EQUAL(grid3->spec_str(), R"({"grid":"H2"})"); for (const std::string& name : {"h2N", "Hn2", "h2_nEsted"}) { std::unique_ptr grid(GridFactory::build(spec::Custom{{"grid", name}})); EXPECT(*grid2 == *grid); } for (const std::string& name : {"H2", "h2r", "hR2", "h2_rinG"}) { std::unique_ptr grid(GridFactory::build(spec::Custom{{"grid", name}})); EXPECT(*grid3 == *grid); } } CASE("sizes") { struct test_t { explicit test_t(size_t N) : N(N), size(12 * N * N) {} size_t N; size_t size; } tests[]{test_t{2}, test_t{3}, test_t{64}}; for (const auto& test : tests) { std::unique_ptr grid1(GridFactory::build(spec::Custom({{"grid", "h" + std::to_string(test.N)}}))); std::unique_ptr grid2(GridFactory::build(spec::Custom({{"type", "HEALPix"}, {"Nside", test.N}}))); grid::reduced::HEALPix grid3(test.N); EXPECT(grid1->size() == test.size); EXPECT(grid2->size() == test.size); EXPECT(grid3.size() == test.size); } } CASE("points") { std::unique_ptr ring(new grid::reduced::HEALPix(2)); EXPECT(ring->order() == order::HEALPix::RING); std::unique_ptr nested(new grid::reduced::HEALPix(2, order::HEALPix::NESTED)); EXPECT(nested->order() == order::HEALPix::NESTED); // reference coordinates in ring order() const std::vector ref{ {45., 66.443535691}, {135., 66.443535691}, {225., 66.443535691}, {315., 66.443535691}, {22.5, 41.810314896}, {67.5, 41.810314896}, {112.5, 41.810314896}, {157.5, 41.810314896}, {202.5, 41.810314896}, {247.5, 41.810314896}, {292.5, 41.810314896}, {337.5, 41.810314896}, {0., 19.471220634}, {45., 19.471220634}, {90., 19.471220634}, {135., 19.471220634}, {180., 19.471220634}, {225., 19.471220634}, {270., 19.471220634}, {315., 19.471220634}, {22.5, 0.}, {67.5, 0.}, {112.5, 0.}, {157.5, 0.}, {202.5, 0.}, {247.5, 0.}, {292.5, 0.}, {337.5, 0.}, {0., -19.471220634}, {45., -19.471220634}, {90., -19.471220634}, {135., -19.471220634}, {180., -19.471220634}, {225., -19.471220634}, {270., -19.471220634}, {315., -19.471220634}, {22.5, -41.810314896}, {67.5, -41.810314896}, {112.5, -41.810314896}, {157.5, -41.810314896}, {202.5, -41.810314896}, {247.5, -41.810314896}, {292.5, -41.810314896}, {337.5, -41.810314896}, {45., -66.443535691}, {135., -66.443535691}, {225., -66.443535691}, {315., -66.443535691}, }; auto points_r = ring->to_points(); EXPECT(points_r.size() == ring->size()); ASSERT(points_r.size() == ref.size()); auto itr = ring->begin(); for (size_t i = 0; i < points_r.size(); ++i) { EXPECT(points_equal(ref[i], points_r[i])); EXPECT(points_equal(ref[i], *itr)); ++itr; } EXPECT(itr == ring->end()); size_t i = 0; for (const auto& it : *ring) { EXPECT(points_equal(ref[i++], it)); } EXPECT(i == ring->size()); auto ren = nested->reorder(order::HEALPix::RING); auto points_n = nested->to_points(); EXPECT(points_n.size() == nested->size()); ASSERT(points_n.size() == ref.size()); auto it = nested->begin(); for (size_t i = 0; i < points_n.size(); ++i) { EXPECT(points_equal(ref[ren.at(i)], points_n[i])); EXPECT(points_equal(ref[ren.at(i)], *it)); ++it; } EXPECT(it == nested->end()); size_t j = 0; for (const auto& it : *nested) { EXPECT(points_equal(ref[ren.at(j++)], it)); } EXPECT(i == nested->size()); } CASE("equals") { std::unique_ptr grid1(GridFactory::build(spec::Custom({{"grid", "h2"}}))); std::unique_ptr grid2(GridFactory::make_from_string("{type: HEALPix, Nside: 2}")); std::unique_ptr grid3(new grid::reduced::HEALPix(2)); EXPECT(*grid1 == *grid2); EXPECT(*grid2 == *grid3); EXPECT(*grid3 == *grid1); EXPECT(grid1->order() == order::HEALPix::RING); std::unique_ptr grid4(GridFactory::build(spec::Custom({{"grid", "h2"}, {"order", "nested"}}))); std::unique_ptr grid5(GridFactory::make_from_string("{type: HEALPix, Nside: 2, order: nested}")); std::unique_ptr grid6(new grid::reduced::HEALPix(2, order::HEALPix::NESTED)); EXPECT(*grid4 != *grid1); EXPECT(*grid4 == *grid5); EXPECT(*grid5 == *grid6); EXPECT(*grid6 == *grid4); EXPECT(grid4->order() == order::HEALPix::NESTED); } CASE("wrong spec") { EXPECT_THROWS_AS(auto* ignore = GridFactory::make_from_string("{grid:h0}"), exception::SpecError); EXPECT_THROWS_AS(auto* ignore = GridFactory::make_from_string("{grid:h3, order:nest}"), exception::OrderError); EXPECT_THROWS_AS(auto* ignore = GridFactory::make_from_string("{grid:h3, order:?}"), exception::OrderError); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/pointlonlat.cc0000664000175000017500000003417515161702250017602 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/Point.h" #include "eckit/geo/PointLonLat.h" #include "eckit/geo/eckit_geo_config.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::test { CASE("PointLonLat normalise_angle_to_*") { struct test_t { double angle; double lim; double ref; }; SECTION("normalise_angle_to_minimum") { for (const auto& test : std::vector{ {10., 0., 10.}, {0., 0., 0.}, {-10., 0., 350.}, {720., 0., 0.}, {100., 90., 100.}, {-370., 0., 350.}, {100000., 0., static_cast(100000 % 360)}, {-100., -180., -100.}, {360., 0., 0.}, {100000., 99960., 100000.}, }) { EXPECT(types::is_approximately_equal( test.ref, PointLonLat::normalise_angle_to_minimum(test.angle, test.lim), PointLonLat::EPS)); } } SECTION("normalise_angle_to_maximum") { for (const auto& test : std::vector{ {350., 360., 350.}, {360., 360., 360.}, {361., 360., 1.}, {-720., 360., 360.}, {100., 180., 100.}, {-370., 360., 350.}, {100000., 360., static_cast(100000 % 360)}, {-100., -90., -100.}, {720., 360., 360.}, {100040., 100080., 100040.}, }) { EXPECT(types::is_approximately_equal( test.ref, PointLonLat::normalise_angle_to_maximum(test.angle, test.lim), PointLonLat::EPS)); } } } CASE("PointLonLat antipode") { PointLonLat p(300., -10.); auto q = p.antipode(); EXPECT(points_equal(q, {120., 10.})); EXPECT(points_equal(p, q.antipode())); PointLonLat r(-10., -91.); EXPECT(points_equal(r.antipode(), {350., 89.})); EXPECT(points_equal(r, r.antipode().antipode())); PointLonLat s(1., -90.); auto t = s.antipode(); EXPECT_EQUAL(t.lon(), 0.); EXPECT(points_equal(t, {2., 90.})); EXPECT(points_equal(t.antipode(), s)); } #if eckit_HAVE_GEO_BITREPRODUCIBLE CASE("bit-identical behaviour normalising angles") { auto normalise = [](double a, double minimum) -> double { auto modulo_360 = [](double a) { return a - 360. * std::floor(a / 360.); }; auto diff = a - minimum; return 0. <= diff && diff < 360. ? a : modulo_360(diff) + minimum; }; struct test_t { double angle; double ref_norm_angle_m_360; double ref_norm_angle_m_720; double ref_norm_angle_p_360; double ref_norm_angle_p_720; }; for (const auto& test : { test_t{0x1.a99999999999fp+3, 0x1.a9999999999ap+3, 0x1.a99999999998p+3, 0x1.a99999999998p+3, 0x1.a99999999998p+3}, // 13.30000000000001 {0x1.7599999999999p+5, 0x1.7599999999998p+5, 0x1.75999999999ap+5, 0x1.75999999999ap+5, 0x1.75999999999ap+5}, // 46.699999999999996 {-0x1.37823af2187f7p+4, -0x1.37823af2187fp+4, -0x1.37823af2188p+4, -0x1.37823af2188p+4, -0x1.37823af2188p+4}, //-19.469294496237094 {0x1.14f26c8adc252p+3, 0x1.14f26c8adc26p+3, 0x1.14f26c8adc24p+3, 0x1.14f26c8adc28p+3, 0x1.14f26c8adc24p+3}, // 8.6545927726848824 {0x1.237e9f537dd2dp+5, 0x1.237e9f537dd3p+5, 0x1.237e9f537dd3p+5, 0x1.237e9f537dd3p+5, 0x1.237e9f537dd3p+5}, // 36.436827327992752 {0x1.eb74b977e1e89p+5, 0x1.eb74b977e1e88p+5, 0x1.eb74b977e1e9p+5, 0x1.eb74b977e1e8p+5, 0x1.eb74b977e1e9p+5}, // 61.431994377690962 {0x1.1008717af4f67p+6, 0x1.1008717af4f68p+6, 0x1.1008717af4f68p+6, 0x1.1008717af4f68p+6, 0x1.1008717af4f68p+6}, // 68.008245392991384 {-0x1.b4f88656270d9p+4, -0x1.b4f88656270ep+4, -0x1.b4f88656270ep+4, -0x1.b4f88656270ep+4, -0x1.b4f88656270ep+4}, //-27.31067498830166 {-0x1.eb22f87f6ac12p+1, -0x1.eb22f87f6acp+1, -0x1.eb22f87f6acp+1, -0x1.eb22f87f6acp+1, -0x1.eb22f87f6acp+1}, //-3.8370047208932272 {0x1.40de11e0c3e99p+4, 0x1.40de11e0c3eap+4, 0x1.40de11e0c3eap+4, 0x1.40de11e0c3eap+4, 0x1.40de11e0c3eap+4}, // 20.054216268529306 {0x1.4aeba99be1331p+5, 0x1.4aeba99be133p+5, 0x1.4aeba99be133p+5, 0x1.4aeba99be133p+5, 0x1.4aeba99be133p+5}, // 41.365069597063105 {0x1.aa5c50f727ae6p+5, 0x1.aa5c50f727ae8p+5, 0x1.aa5c50f727aep+5, 0x1.aa5c50f727aep+5, 0x1.aa5c50f727aep+5}, // 53.295076304338906 {-0x1.556ccf04ef1bbp+4, -0x1.556ccf04ef1cp+4, -0x1.556ccf04ef1cp+4, -0x1.556ccf04ef1cp+4, -0x1.556ccf04ef1cp+4}, //-21.339064616464139 {0x1.556ccf04ef1bbp+4, 0x1.556ccf04ef1cp+4, 0x1.556ccf04ef1cp+4, 0x1.556ccf04ef1cp+4, 0x1.556ccf04ef1cp+4}, // 21.339064616464139 {0x1.388f683df92bbp+5, 0x1.388f683df92b8p+5, 0x1.388f683df92cp+5, 0x1.388f683df92cp+5, 0x1.388f683df92cp+5}, // 39.070023044745049 {-0x1.40de11e0c3e9dp+4, -0x1.40de11e0c3eap+4, -0x1.40de11e0c3eap+4, -0x1.40de11e0c3eap+4, -0x1.40de11e0c3eap+4}, //-20.05421626852932 {0x1.eb22f87f6abf5p+1, 0x1.eb22f87f6acp+1, 0x1.eb22f87f6acp+1, 0x1.eb22f87f6acp+1, 0x1.eb22f87f6acp+1}, // 3.8370047208932143 {0x1.b4f88656270d7p+4, 0x1.b4f88656270dp+4, 0x1.b4f88656270ep+4, 0x1.b4f88656270cp+4, 0x1.b4f88656270ep+4}, // 27.310674988301653 {-0x1.3f0f4411db559p+5, -0x1.3f0f4411db558p+5, -0x1.3f0f4411db56p+5, -0x1.3f0f4411db558p+5, -0x1.3f0f4411db56p+5}, //-39.882454051500368 {-0x1.63664f7d2181dp+5, -0x1.63664f7d2182p+5, -0x1.63664f7d2182p+5, -0x1.63664f7d2182p+5, -0x1.63664f7d2182p+5}, //-44.424956300339751 {-0x1.75e470fd085aap+5, -0x1.75e470fd085a8p+5, -0x1.75e470fd085bp+5, -0x1.75e470fd085a8p+5, -0x1.75e470fd085bp+5}, //-46.7365436332869 {-0x1.b2a6314996231p+4, -0x1.b2a631499623p+4, -0x1.b2a631499624p+4, -0x1.b2a631499624p+4, -0x1.b2a631499624p+4}, //-27.165574347922476 {-0x1.f720e2a9525edp+5, -0x1.f720e2a9525fp+5, -0x1.f720e2a9525fp+5, -0x1.f720e2a9525fp+5, -0x1.f720e2a9525fp+5}, //-62.89105732233643 {-0x1.236723c039272p+5, -0x1.236723c03927p+5, -0x1.236723c03927p+5, -0x1.236723c03927p+5, -0x1.236723c03927p+5}, //-36.425361158126989 {-0x1.7f9f1a40a5d1fp+4, -0x1.7f9f1a40a5d2p+4, -0x1.7f9f1a40a5d2p+4, -0x1.7f9f1a40a5d2p+4, -0x1.7f9f1a40a5d2p+4}, //-23.976343395738805 {0x1.ffffffffffffep+0, 0x1p+1, 0x1p+1, 0x1p+1, 0x1p+1}, // 1.9999999999999996 {0x1.0b907154a92f7p+6, 0x1.0b907154a92f8p+6, 0x1.0b907154a92f8p+6, 0x1.0b907154a92f8p+6, 0x1.0b907154a92f8p+6}, // 66.891057322336437 {0x1.436723c039272p+5, 0x1.436723c03927p+5, 0x1.436723c03927p+5, 0x1.436723c03927p+5, 0x1.436723c03927p+5}, // 40.425361158126989 {0x1.bf9f1a40a5d1fp+4, 0x1.bf9f1a40a5d2p+4, 0x1.bf9f1a40a5d2p+4, 0x1.bf9f1a40a5d2p+4, 0x1.bf9f1a40a5d2p+4}, // 27.976343395738805 {0x1.0f266c20b79f9p+7, 0x1.0f266c20b79f8p+7, 0x1.0f266c20b79f8p+7, 0x1.0f266c20b79f8p+7, 0x1.0f266c20b79f8p+7}, // 135.57504369966026 {0x1.787bbbb54c676p+6, 0x1.787bbbb54c678p+6, 0x1.787bbbb54c678p+6, 0x1.787bbbb54c678p+6, 0x1.787bbbb54c678p+6}, // 94.120833237446135 {0x1.95e470fd085aap+5, 0x1.95e470fd085a8p+5, 0x1.95e470fd085bp+5, 0x1.95e470fd085ap+5, 0x1.95e470fd085bp+5}, // 50.7365436332869 {0x1.1bd0dd7b42b69p+7, 0x1.1bd0dd7b42b68p+7, 0x1.1bd0dd7b42b68p+7, 0x1.1bd0dd7b42b68p+7, 0x1.1bd0dd7b42b68p+7}, // 141.90793976964349 {0x1.19981bd70b549p+6, 0x1.19981bd70b548p+6, 0x1.19981bd70b548p+6, 0x1.19981bd70b548p+6, 0x1.19981bd70b548p+6}, // 70.39854370123534 {0x1.50bc8a12f525bp+5, 0x1.50bc8a12f5258p+5, 0x1.50bc8a12f526p+5, 0x1.50bc8a12f526p+5, 0x1.50bc8a12f526p+5}, // 42.092060230356502 {0x1.cb2a2664f7bbdp+6, 0x1.cb2a2664f7bbcp+6, 0x1.cb2a2664f7bcp+6, 0x1.cb2a2664f7bcp+6, 0x1.cb2a2664f7bcp+6}, // 114.79116208803221 {0x1.6784444ab398ap+6, 0x1.6784444ab3988p+6, 0x1.6784444ab3988p+6, 0x1.6784444ab3988p+6, 0x1.6784444ab3988p+6}, // 89.879166762553865 {0x1.83664f7d2181ep+5, 0x1.83664f7d2182p+5, 0x1.83664f7d2182p+5, 0x1.83664f7d2182p+5, 0x1.83664f7d2182p+5}, // 48.424956300339758 {0x1.380c1cb7eb45dp+7, 0x1.380c1cb7eb45cp+7, 0x1.380c1cb7eb45cp+7, 0x1.380c1cb7eb45cp+7, 0x1.380c1cb7eb46p+7}, // 156.02365660426122 {0x1.d46f8eab56d0bp+6, 0x1.d46f8eab56d0cp+6, 0x1.d46f8eab56d08p+6, 0x1.d46f8eab56d1p+6, 0x1.d46f8eab56d08p+6}, // 117.10894267766359 {0x1.5f0f4411db559p+5, 0x1.5f0f4411db558p+5, 0x1.5f0f4411db56p+5, 0x1.5f0f4411db56p+5, 0x1.5f0f4411db56p+5}, // 43.882454051500368 }) { EXPECT(test.angle == normalise(test.angle, -180.)); EXPECT(test.ref_norm_angle_m_360 == normalise(test.angle - 360., -180.)); EXPECT(test.ref_norm_angle_m_720 == normalise(test.angle - 720., -180.)); EXPECT(test.ref_norm_angle_p_360 == normalise(test.angle + 360., -180.)); EXPECT(test.ref_norm_angle_p_720 == normalise(test.angle + 720., -180.)); } } #endif CASE("PointLonLat normalisation") { constexpr auto da = 1e-3; for (double a : {-370., -190., -100., -90., -80., -10., 0., 10., 80., 90., 100., 190., 370.}) { EXPECT(types::is_approximately_equal(a, PointLonLat::normalise_angle_to_minimum(a, a), PointLonLat::EPS)); EXPECT(types::is_approximately_equal(a, PointLonLat::normalise_angle_to_maximum(a, a), PointLonLat::EPS)); EXPECT(types::is_approximately_equal(a + 360., PointLonLat::normalise_angle_to_minimum(a - da, a) + da, PointLonLat::EPS)); EXPECT(types::is_approximately_equal(a - 360., PointLonLat::normalise_angle_to_maximum(a + da, a) - da, PointLonLat::EPS)); } PointLonLat p(1, 90.); EXPECT_EQUAL(p.lon(), 1.); EXPECT_EQUAL(p.lat(), 90.); auto p2 = PointLonLat::make(p.lon(), p.lat()); EXPECT_EQUAL(p2.lon(), 0.); EXPECT(points_equal(p, p2)); auto p3 = PointLonLat(50., 90.); EXPECT(points_equal(p, p3)); } CASE("PointLonLat comparison") { Point a1 = PointLonLat{180., 0.}; Point a2 = PointLonLat{-180., 0.}; EXPECT(points_equal(a1, a2)); Point b1 = PointLonLat{0., -90.}; Point b2 = PointLonLat::make(1., 270.); EXPECT(points_equal(b1, b2)); Point c1 = PointLonLat{300, -30}; Point c2 = PointLonLat{-59.99999999999996, -30.000000000000018}; EXPECT(points_equal(c1, c2)); Point d1 = PointLonLat{-178., -46.7}; Point d2 = PointLonLat{-178.00000000000003, -46.7}; EXPECT(points_equal(d1, d2)); Point e1 = PointLonLat(0., 90.); Point e2 = PointLonLat(180., 90.); EXPECT(points_equal(e1, e2)); Point f1 = PointLonLat(-0., -90.); Point f2 = PointLonLat(-180., -90.); EXPECT(points_equal(f1, f2)); } CASE("PointLonLat normalise angles") { EXPECT(0. == PointLonLat::normalise_angle_to_minimum(360., 0.)); EXPECT(14. == PointLonLat::normalise_angle_to_minimum(374., 0.)); EXPECT(14. == PointLonLat::normalise_angle_to_minimum(374., -90.)); EXPECT(374. == PointLonLat::normalise_angle_to_minimum(374., 90.)); EXPECT(14. == PointLonLat::normalise_angle_to_minimum(-346., 0.)); EXPECT(14. == PointLonLat::normalise_angle_to_minimum(-346., -90.)); EXPECT(374. == PointLonLat::normalise_angle_to_minimum(-346., 90.)); EXPECT(0. == PointLonLat::normalise_angle_to_minimum(360. * 1e12, 0.)); EXPECT(14. == PointLonLat::normalise_angle_to_minimum(360. * 1e12 + 14, 0.)); EXPECT(0. == PointLonLat::normalise_angle_to_minimum(-360. * 1e12, 0.)); EXPECT(14. == PointLonLat::normalise_angle_to_minimum(-360. * 1e12 + 14, 0.)); } CASE("PointLonLat canonicalise on sphere") { const PointLonLat p1(108., 32.); // Worse coordinates for the same point: const auto p2 = PointLonLat::make(-252., 32.); const auto p3 = PointLonLat::make(288., 148.); const auto p4 = PointLonLat::make(108., -328.); // Check each of these is correctly shifted back to original point: const PointLonLat q2 = PointLonLat::make(p2.lon(), p2.lat()); const PointLonLat q3 = PointLonLat::make(p3.lon(), p3.lat()); const PointLonLat q4 = PointLonLat::make(p4.lon(), p4.lat()); EXPECT(p1.lon() == q2.lon()); EXPECT(p1.lat() == q2.lat()); EXPECT(p1.lon() == q3.lon()); EXPECT(p1.lat() == q3.lat()); EXPECT(p1.lon() == q4.lon()); EXPECT(p1.lat() == q4.lat()); // Check with longitude offset EXPECT(points_equal(PointLonLat::make(1., -90.), PointLonLat(0., -90.))); EXPECT(points_equal(PointLonLat::make(2., 90.), PointLonLat(0., 90.))); EXPECT(points_equal(PointLonLat::make(3., 180.), PointLonLat(183., 0.))); // Check with latitude offset constexpr double eps = 1.e-6; EXPECT(points_equal(PointLonLat::make(-1., 89.99999914622634, 0., eps), {0., 90.})); EXPECT(points_equal(PointLonLat::make(1., -89.99999914622634, 0., eps), {0., -90.})); } CASE("PointLonLat pole") { for (const auto& p : {PointLonLat(0., 90.), PointLonLat(0., -90.)}) { EXPECT(p.pole()); PointLonLat q{p.lon(), p.lat() + 1.}; EXPECT(!q.pole()); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/order.cc0000664000175000017500000000412715161702250016344 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/order/HEALPix.h" #include "eckit/geo/order/Scan.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("order=scan") { for (std::string scan : {"i+j+", "i+j-"}) { order::Scan order(scan); EXPECT(scan == order.order()); } } CASE("healpix") { using order::HEALPix; auto is_power_of_two = [](size_t n) { return (n & (n - 1)) == 0; }; SECTION("order=ring/nested") { size_t Nside = 4; EXPECT(HEALPix::order_default() == HEALPix(spec::Custom{}).order()); EXPECT(HEALPix::RING == HEALPix(spec::Custom{{"order", HEALPix::RING}}).order()); EXPECT(HEALPix::NESTED == HEALPix(spec::Custom{{"order", HEALPix::NESTED}}).order()); for (const auto& type : {HEALPix::order_default(), HEALPix::RING, HEALPix::NESTED}) { HEALPix order(type); auto no_reorder = order.reorder(type, Nside); std::vector expected(no_reorder.size()); std::iota(expected.begin(), expected.end(), 0); EXPECT(expected == no_reorder); } } SECTION("exceptions") { EXPECT_THROWS_AS(HEALPix(spec::Custom{{"order", "?"}}), exception::OrderError); EXPECT_THROWS_AS(HEALPix("?"), exception::OrderError); for (size_t Nside : {1, 2, 3}) { const auto size = 12 * Nside * Nside; EXPECT_NO_THROW(HEALPix{}); EXPECT_NO_THROW(HEALPix{HEALPix::order_default()}); } } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/projection_plate-caree.cc0000664000175000017500000000216215161702250021644 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/Projection.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { using P = std::unique_ptr; CASE("projection: plate-caree") { Point p = PointXY{1, 1}; Point q = PointLonLat{1, 1}; P projection(ProjectionFactoryType::instance().get("plate-carree").create(spec::Custom{})); EXPECT(points_equal(q, projection->inv(p))); EXPECT(std::holds_alternative(projection->inv(p))); EXPECT(points_equal(p, projection->fwd(q))); EXPECT(std::holds_alternative(projection->fwd(q))); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/util.cc0000664000175000017500000001524015161702250016204 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include "eckit/geo/util.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" template bool is_approximately_equal_vector(const std::vector& a, const std::vector& b, double eps = 0.) { return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin(), [=](const T& a, const T& b) { return ::eckit::types::is_approximately_equal(a, b, eps); }); } template std::ostream& operator<<(std::ostream& out, const std::vector& v) { const auto* sep = ""; for (const auto& value : v) { out << sep << value; sep = ", "; } return out; } namespace eckit::geo::test { constexpr double EPS = 1e-9; CASE("eckit::geo::util::linspace") { EXPECT(is_approximately_equal_vector(util::linspace(1, 2, 1), std::vector{1.}, EPS)); EXPECT(is_approximately_equal_vector(util::linspace(1, 1, 2), std::vector{1.}, EPS)); EXPECT(is_approximately_equal_vector(util::linspace(1, 2, 2), std::vector{1., 2.}, EPS)); } CASE("eckit::geo::util::arange") { EXPECT(is_approximately_equal_vector(util::arange(1, 2, 0.5), std::vector{1, 1.5, 2}, EPS)); } CASE("eckit::geo::util::gaussian_latitudes") { std::vector lats_decreasing{ 88.9277, 87.5387, 86.1415, 84.7424, 83.3426, 81.9425, 80.5421, 79.1417, 77.7412, 76.3406, 74.94, 73.5394, 72.1387, 70.7381, 69.3374, 67.9367, 66.536, 65.1353, 63.7345, 62.3338, 60.9331, 59.5323, 58.1316, 56.7309, 55.3301, 53.9294, 52.5286, 51.1279, 49.7271, 48.3264, 46.9256, 45.5249, 44.1241, 42.7233, 41.3226, 39.9218, 38.5211, 37.1203, 35.7195, 34.3188, 32.918, 31.5172, 30.1165, 28.7157, 27.315, 25.9142, 24.5134, 23.1127, 21.7119, 20.3111, 18.9104, 17.5096, 16.1088, 14.7081, 13.3073, 11.9065, 10.5058, 9.10499, 7.70422, 6.30345, 4.90269, 3.50192, 2.10115, 0.700384, -0.700384, -2.10115, -3.50192, -4.90269, -6.30345, -7.70422, -9.10499, -10.5058, -11.9065, -13.3073, -14.7081, -16.1088, -17.5096, -18.9104, -20.3111, -21.7119, -23.1127, -24.5134, -25.9142, -27.315, -28.7157, -30.1165, -31.5172, -32.918, -34.3188, -35.7195, -37.1203, -38.5211, -39.9218, -41.3226, -42.7233, -44.1241, -45.5249, -46.9256, -48.3264, -49.7271, -51.1279, -52.5286, -53.9294, -55.3301, -56.7309, -58.1316, -59.5323, -60.9331, -62.3338, -63.7345, -65.1353, -66.536, -67.9367, -69.3374, -70.7381, -72.1387, -73.5394, -74.94, -76.3406, -77.7412, -79.1417, -80.5421, -81.9425, -83.3426, -84.7424, -86.1415, -87.5387, -88.9277}; std::vector lats_increasing(lats_decreasing); std::reverse(lats_increasing.begin(), lats_increasing.end()); EXPECT(is_approximately_equal_vector(util::gaussian_latitudes(64, false), lats_decreasing, 1e-4)); EXPECT(is_approximately_equal_vector(util::gaussian_latitudes(64, true), lats_increasing, 1e-4)); } CASE("eckit::geo::util::gaussian_quadrature_weights") { std::vector weights{ 0.000225, 0.000523, 0.000821, 0.001119, 0.001416, 0.001713, 0.002008, 0.002302, 0.002595, 0.002886, 0.003176, 0.003463, 0.003749, 0.004032, 0.004313, 0.004592, 0.004867, 0.005140, 0.005409, 0.005676, 0.005939, 0.006198, 0.006454, 0.006706, 0.006953, 0.007197, 0.007437, 0.007672, 0.007902, 0.008128, 0.008348, 0.008564, 0.008775, 0.008980, 0.009180, 0.009375, 0.009564, 0.009747, 0.009924, 0.010096, 0.010261, 0.010421, 0.010574, 0.010721, 0.010861, 0.010995, 0.011122, 0.011243, 0.011357, 0.011464, 0.011564, 0.011658, 0.011744, 0.011823, 0.011896, 0.011961, 0.012019, 0.012070, 0.012114, 0.012150, 0.012179, 0.012201, 0.012216, 0.012223, 0.012223, 0.012216, 0.012201, 0.012179, 0.012150, 0.012114, 0.012070, 0.012019, 0.011961, 0.011896, 0.011823, 0.011744, 0.011658, 0.011564, 0.011464, 0.011357, 0.011243, 0.011122, 0.010995, 0.010861, 0.010721, 0.010574, 0.010421, 0.010261, 0.010096, 0.009924, 0.009747, 0.009564, 0.009375, 0.009180, 0.008980, 0.008775, 0.008564, 0.008348, 0.008128, 0.007902, 0.007672, 0.007437, 0.007197, 0.006953, 0.006706, 0.006454, 0.006198, 0.005939, 0.005676, 0.005409, 0.005140, 0.004867, 0.004592, 0.004313, 0.004032, 0.003749, 0.003463, 0.003176, 0.002886, 0.002595, 0.002302, 0.002008, 0.001713, 0.001416, 0.001119, 0.000821, 0.000523, 0.000225}; EXPECT(is_approximately_equal_vector(util::gaussian_quadrature_weights(64), weights, 1e-4)); } CASE("eckit::geo::util::reduced_classical_pl") { const pl_type pl{20, 27, 32, 40, 45, 48, 60, 60, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 60, 60, 48, 45, 40, 32, 27, 20}; EXPECT(is_approximately_equal_vector(util::reduced_classical_pl(16), pl)); EXPECT(is_approximately_equal_vector(util::reduced_classical_pl(16), pl)); } CASE("eckit::geo::util::reduced_octahedral_pl") { const pl_type pl{20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 80, 76, 72, 68, 64, 60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20}; EXPECT(is_approximately_equal_vector(util::reduced_octahedral_pl(16), pl)); EXPECT(is_approximately_equal_vector(util::reduced_octahedral_pl(16), pl)); } CASE("eckit::geo::util::monotonic_crop") { struct test { double min; double max; std::pair expected; std::vector values; std::vector cropped; } tests[]{ {1., 1., {0, 1}, {1.}, {1.}}, {1., 2., {0, 3}, {1., 1., 1.}, {1., 1., 1.}}, {2., 3., {1, 3}, {1., 2., 3., 4., 5., 6.}, {2., 3.}}, {2., 3., {3, 5}, {6., 5., 4., 3., 2., 1.}, {3., 2.}}, }; for (const auto& test : tests) { auto [from, to] = util::monotonic_crop(test.values, test.min, test.max); EXPECT(from == test.expected.first); EXPECT(to == test.expected.second); EXPECT(is_approximately_equal_vector(std::vector(test.values.begin() + from, test.values.begin() + to), test.cropped)); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/point3.cc0000664000175000017500000000437415161702250016451 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/PointXYZ.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::test { CASE("PointXYZ initialisation") { PointXYZ z; EXPECT(z.X() == 0.); EXPECT(z.Y() == 0.); EXPECT(z.Z() == 0.); PointXYZ p = {1., 2., 3.}; PointXYZ s(p); EXPECT(s.X() == 1.); EXPECT(s.Y() == 2.); EXPECT(s.Z() == 3.); } CASE("PointXYZ addition") { PointXYZ p1 = {1., 2., 3.}; PointXYZ p2{1., 2., 3.}; PointXYZ r = p1 + p2; EXPECT(r.X() == 2.); EXPECT(r.Y() == 4.); EXPECT(r.Z() == 6.); } CASE("PointXYZ subtraction") { PointXYZ p1{2., 5., 7.}; PointXYZ p2{1., 2., 3.}; PointXYZ r = p1 - p2; EXPECT(r.X() == 1.); EXPECT(r.Y() == 3.); EXPECT(r.Z() == 4.); } CASE("PointXYZ scaling") { PointXYZ p1{1., 2., 3.}; PointXYZ r = p1 * 42.; EXPECT(r.X() == 42.); EXPECT(r.Y() == 84.); EXPECT(r.Z() == 126.); } CASE("PointXYZ equality") { PointXYZ p1{1., 2., 3.}; PointXYZ p2{1., 2., 3.}; EXPECT(p1 == p2); } CASE("PointXYZ inequality") { PointXYZ p1{1., 2., 3.}; PointXYZ p2{1., 2., 4.}; EXPECT(p1 != p2); } CASE("PointXYZ distance comparison") { PointXYZ p1{2., 1., 0.}; PointXYZ p2{1., 2., 4.}; PointXYZ p3{5., 5., 5.}; EXPECT(types::is_approximately_equal(std::sqrt(18.), p1.distance(p2))); EXPECT(types::is_approximately_equal(std::sqrt(50.), p1.distance(p3))); } CASE("PointXYZ distance2 comparison") { PointXYZ p1{2., 1., 0.}; PointXYZ p2{1., 2., 4.}; PointXYZ p3; EXPECT(types::is_approximately_equal(p1.distance2(p2), 18.)); EXPECT(types::is_approximately_equal(p1.distance2(p3), 5.)); EXPECT(p2.distance2(p1) > p3.distance2(p1)); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/projection_mercator.cc0000664000175000017500000000425415161702250021302 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/eckit_geo_config.h" #include "eckit/geo/projection/Mercator.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("Mercator: spec_str, proj_str") { constexpr auto eps = 10. * PointXY::EPS; // FIXME improve floating-point errors projection::Mercator proj1({0., 14.}, {262.036, 14.7365}); projection::Mercator proj2({0., 14.}, {0., 0.}); projection::Mercator proj3({-180., 0.}, {0., 0.}); SECTION("inv(fwd(.)) == . and fwd(inv(.)) == .") { for (const auto& projection : { proj1, proj2, proj3, }) { PointXY a{0., 0.}; EXPECT(points_equal(a, projection.fwd(projection.inv(a)), eps)); PointLonLat b{-75., 35.}; EXPECT(points_equal(b, projection.inv(projection.fwd(b)), eps)); PointXY c = projection.fwd(NORTH_POLE); EXPECT(c.Y() > std::numeric_limits::max()); PointXY d = projection.fwd(SOUTH_POLE); EXPECT(d.Y() < std::numeric_limits::lowest()); } } SECTION("spec_str") { EXPECT(proj1.spec_str() == R"({"lat_ts":14,"r":6371229,"type":"mercator"})"); EXPECT(proj2.spec_str() == R"({"lat_ts":14,"r":6371229,"type":"mercator"})"); EXPECT(proj3.spec_str() == R"({"lon_0":-180,"r":6371229,"type":"mercator"})"); } #if eckit_HAVE_PROJ SECTION("proj_str") { EXPECT(proj1.proj_str() == "+proj=merc +lat_ts=14 +R=6371229"); EXPECT(proj2.proj_str() == "+proj=merc +lat_ts=14 +R=6371229"); EXPECT(proj3.proj_str() == "+proj=merc +lon_0=-180 +R=6371229"); } #endif } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/pointlonlatr.cc0000664000175000017500000001137315161702250017757 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/PointLonLat.h" #include "eckit/geo/PointLonLatR.h" #include "eckit/geo/util.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::test { CASE("PointLonLatR normalise_angle_to_*") { struct test_t { double angle; double lim; double ref; }; SECTION("normalise_angle_to_minimum") { for (const auto& test : std::vector{ {1., 0., 1.}, {1. + 42. * PointLonLatR::FULL_ANGLE, 0., 1.}, {1. - 42. * PointLonLatR::FULL_ANGLE, 0., 1.}, {1., 3. * PointLonLatR::FULL_ANGLE, 3. * PointLonLatR::FULL_ANGLE + 1.}, {-1., 3. * PointLonLatR::FULL_ANGLE, 4. * PointLonLatR::FULL_ANGLE - 1.}, }) { EXPECT(types::is_approximately_equal( test.ref, PointLonLatR::normalise_angle_to_minimum(test.angle, test.lim), PointLonLatR::EPS)); } } SECTION("normalise_angle_to_maximum") { for (const auto& test : std::vector{ {1., 0., 1. - PointLonLatR::FULL_ANGLE}, {1., 3. * PointLonLatR::FULL_ANGLE, 2. * PointLonLatR::FULL_ANGLE + 1.}, {-1., 3. * PointLonLatR::FULL_ANGLE, 3. * PointLonLatR::FULL_ANGLE - 1.}, }) { EXPECT(types::is_approximately_equal( test.ref, PointLonLatR::normalise_angle_to_maximum(test.angle, test.lim), PointLonLatR::EPS)); } } } CASE("PointLonLatR normalisation") { PointLonLatR p(1, PointLonLatR::RIGHT_ANGLE); auto p2 = PointLonLatR::make(p.lonr(), p.latr()); auto p3 = PointLonLatR(1. + 42. * PointLonLatR::FULL_ANGLE, PointLonLatR::RIGHT_ANGLE); EXPECT_EQUAL(p2.lonr(), 0.); EXPECT(points_equal(p, p2)); EXPECT(points_equal(p, p3)); PointLonLatR q(1., SOUTH_POLE_R.latr()); auto q2 = q.antipode(); auto q3 = q2.antipode(); EXPECT_EQUAL(q2.lonr(), 0.); EXPECT(points_equal(q2, p)); EXPECT(points_equal(q3, q)); } CASE("PointLonLatR comparison") { auto r(PointLonLatR::make(-10., -91.)); EXPECT(points_equal(r, r.antipode().antipode())); PointLonLatR a1{PointLonLatR::FLAT_ANGLE, 0.}; PointLonLatR a2{-PointLonLatR::FLAT_ANGLE, 0.}; EXPECT(points_equal(a1, a2)); EXPECT(points_equal(SOUTH_POLE_R, {1., SOUTH_POLE_R.latr() + PointLonLatR::FULL_ANGLE})); PointLonLatR c1{300., -30.}; PointLonLatR c2{c1.lonr() - PointLonLatR::FULL_ANGLE - PointLonLatR::EPS / 10., c1.latr() + PointLonLatR::FULL_ANGLE + PointLonLatR::EPS / 10.}; EXPECT(points_equal(c1, c2)); EXPECT(points_equal(NORTH_POLE_R, {-42, PointLonLatR::RIGHT_ANGLE})); EXPECT(points_equal(SOUTH_POLE_R, {42., -PointLonLatR::RIGHT_ANGLE})); } CASE("PointLonLatR normalise angles") { EXPECT(types::is_approximately_equal( 0., PointLonLatR::normalise_angle_to_minimum(0. + PointLonLatR::FULL_ANGLE, 0.), PointLonLatR::EPS)); EXPECT(types::is_approximately_equal( 1., PointLonLatR::normalise_angle_to_minimum(1. + PointLonLatR::FULL_ANGLE * 11, 0.), PointLonLatR::EPS)); EXPECT(types::is_approximately_equal( 2. + PointLonLatR::FULL_ANGLE * 11, PointLonLatR::normalise_angle_to_minimum(2. + PointLonLatR::FULL_ANGLE * 11, PointLonLatR::FULL_ANGLE * 11), PointLonLatR::EPS)); } CASE("PointLonLatR conversion to/from PointLonLat") { PointLonLatR p{0., 0.}; EXPECT(points_equal(p, PointLonLatR::make_from_lonlat(0., 0.))); PointLonLatR q{0., PointLonLatR::RIGHT_ANGLE}; EXPECT(points_equal(q, PointLonLatR::make_from_lonlat(1., PointLonLat::RIGHT_ANGLE))); PointLonLatR r{42. * PointLonLatR::FULL_ANGLE, SOUTH_POLE_R.latr()}; EXPECT(points_equal(r, PointLonLatR::make_from_lonlat(0., SOUTH_POLE.lat() - 42. * PointLonLat::FULL_ANGLE))); PointLonLatR s{10. * util::DEGREE_TO_RADIAN, 42. * PointLonLatR::FULL_ANGLE}; EXPECT(points_equal(s, PointLonLatR::make_from_lonlat(10. - 42. * PointLonLat::FULL_ANGLE, 0.))); } CASE("PointLonLatR pole") { for (const auto& p : {PointLonLatR(0., PointLonLatR::RIGHT_ANGLE), PointLonLatR(0., -PointLonLatR::RIGHT_ANGLE)}) { EXPECT(p.pole()); PointLonLatR q{p.lonr(), p.latr() + 1.}; EXPECT(!q.pole()); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/cache.cc0000664000175000017500000001472615161702250016302 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit_config.h" #include #include #include "eckit/filesystem/PathName.h" #include "eckit/geo/cache/Download.h" #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/cache/Unzip.h" #include "eckit/geo/util.h" #include "eckit/utils/StringTools.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { const std::string URL = "https://www.ecmwf.int/robots.txt"; CASE("eckit::geo::util") { using Cache = cache::MemoryCache; struct test_t { size_t N; bool increasing; Cache::bytes_size_t pl_footprint; Cache::bytes_size_t pl_footprint_acc; Cache::bytes_size_t gl_footprint; Cache::bytes_size_t gl_footprint_acc; } tests[] = { {16, false, 256, 256, 256, 256}, // {24, false, 384, 640, 384, 640}, // {24, false, 384, 640, 384, 640}, // (repeated for a cache hit) {32, false, 512, 1152, 512, 1152}, // {16, false, 256, 1152, 256, 1152}, // (repeated for another cache hit) {48, false, 768, 1920, 768, 1920}, // {16, true, 256, 1920, 256, 2176}, // (repeated except for 'increasing') {24, true, 384, 1920, 384, 2560}, // ... {24, true, 384, 1920, 384, 2560}, // {32, true, 512, 1920, 512, 3072}, // {16, true, 256, 1920, 256, 3072}, // {48, true, 768, 1920, 768, 3840}, // }; Cache::total_purge(); EXPECT_EQUAL(0, Cache::total_footprint()); SECTION("separate caches") { util::reduced_classical_pl(16); auto foot = Cache::total_footprint(); EXPECT(0 < foot); util::reduced_octahedral_pl(16); EXPECT(foot < Cache::total_footprint()); } Cache::total_purge(); EXPECT_EQUAL(0, Cache::total_footprint()); SECTION("reduced_classical_pl, reduced_octahedral_pl") { for (const auto& fun : {util::reduced_classical_pl, util::reduced_octahedral_pl}) { for (const auto& test : tests) { Cache::total_purge(); fun(test.N); EXPECT_EQUAL(Cache::total_footprint(), test.pl_footprint); } Cache::total_purge(); for (const auto& test : tests) { fun(test.N); EXPECT_EQUAL(Cache::total_footprint(), test.pl_footprint_acc); } } } Cache::total_purge(); EXPECT_EQUAL(0, Cache::total_footprint()); SECTION("gaussian_latitudes") { for (const auto& test : tests) { Cache::total_purge(); util::gaussian_latitudes(test.N, test.increasing); EXPECT_EQUAL(Cache::total_footprint(), test.gl_footprint); } Cache::total_purge(); for (const auto& test : tests) { util::gaussian_latitudes(test.N, test.increasing); EXPECT_EQUAL(Cache::total_footprint(), test.gl_footprint_acc); } } Cache::total_purge(); EXPECT_EQUAL(0, Cache::total_footprint()); } #if eckit_HAVE_CURL CASE("download: error handling") { const PathName path("test.download"); if (path.exists()) { path.unlink(); ASSERT(!path.exists()); } EXPECT_THROWS_AS(cache::Download::to_path("https://does.not/exist", path), UserError); EXPECT(!path.exists()); EXPECT_THROWS_AS(cache::Download::to_path("https://sites.ecmwf.int/repository/does/not/exist", path), UserError); EXPECT(!path.exists()); } CASE("download: non-cached") { const PathName path("test.download"); if (path.exists()) { path.unlink(); ASSERT(!path.exists()); } auto info = cache::Download::to_path(URL, path); EXPECT(info.bytes.value() > 0.); EXPECT(path.exists()); path.unlink(); ASSERT(!path.exists()); } CASE("download: cached") { const std::string prefix = "prefix-"; const std::string suffix = ".suffix"; const PathName root("test.download.dir", true); cache::Download download(root); EXPECT(root == download.cache_root()); download.rm_cache_root(); EXPECT(!root.exists()); auto path = download.to_cached_path(URL, prefix, suffix); EXPECT(root.exists() && root.isDir()); EXPECT(path.exists()); std::string basename = path.baseName(); EXPECT(StringTools::startsWith(basename, prefix)); EXPECT(StringTools::endsWith(basename, suffix)); EXPECT(path.dirName() == root); download.rm_cache_root(); EXPECT(!root.exists()); } #endif #if eckit_HAVE_ZIP CASE("unzip") { const PathName zip(ZIP_FILE); ASSERT(zip.exists()); const std::vector contents{"a", "b/", "b/c"}; SECTION("unzip all") { const PathName dir("eckit_geo_cache/unzip/unzip-all", true); cache::Unzip unzip(dir); unzip.rm_cache_root(); cache::Unzip::to_path(zip, dir); for (const auto& content : contents) { EXPECT((dir / content).exists()); } EXPECT(dir.exists()); unzip.rm_cache_root(); EXPECT(!dir.exists()); for (const auto& what : {"a", "b/c"}) { auto cached_path = unzip.to_cached_path(zip, "a"); EXPECT(dir.exists() && dir.isDir()); EXPECT(cached_path.exists() && !cached_path.isDir()); } } SECTION("unzip one") { const PathName dir("cache.unzip.one", true); cache::Unzip unzip(dir); unzip.rm_cache_root(); cache::Unzip::to_path(zip, dir / (contents.back() + "-y"), contents.back()); for (const auto& content : contents) { if (content == contents.back()) { const PathName file = dir / (contents.back() + "-y"); EXPECT(file.exists()); std::string d; std::ifstream(file.localPath()) >> d; EXPECT(d == "d"); } else { PathName path = dir / content; EXPECT(!path.exists() || path.isDir()); } } unzip.rm_cache_root(); ASSERT(!dir.exists()); } } #endif } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/grid_icon.cc0000664000175000017500000000675315161702250017175 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to itr by virtue of its status as an intergovernmental organisation nor * does itr submit to any jurisdiction. */ #include #include #include #include "eckit/geo/LibEcKitGeo.h" #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/grid/unstructured/ICON.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { static const std::string GRID = "icon-grid-0055-r02b05-n"; static const Grid::uid_type UID = "e234e01a8556e9a84bcb42361d2f24e0"; static const std::vector SHAPE{2656}; CASE("caching") { if (LibEcKitGeo::caching()) { using Cache = cache::MemoryCache; SECTION("Grid::build_from_uid") { spec::Custom spec{{"uid", UID}}; const auto footprint_1 = Cache::total_footprint(); std::cout << spec << std::endl; std::unique_ptr grid1(GridFactory::build(spec)); const auto footprint_2 = Cache::total_footprint(); EXPECT(footprint_1 < footprint_2); std::unique_ptr grid2(GridFactory::build(spec)); EXPECT(footprint_2 == Cache::total_footprint()); EXPECT(grid1->size() == grid2->size()); } } } CASE("spec") { std::unique_ptr spec(GridFactory::make_spec(spec::Custom({{"uid", UID}}))); EXPECT(spec->get_string("type") == "ICON"); EXPECT(spec->get_string("name") == GRID); EXPECT(spec->get_string("icon_number_of_grid_used") == "55"); EXPECT(spec->get_string("icon_type") == "hrz_regional"); EXPECT(spec->get_string("icon_arrangement") == "C"); EXPECT(spec->get_string("icon_uid") == UID); EXPECT(spec->get_long_vector("shape") == SHAPE); std::unique_ptr grid1(GridFactory::make_from_string("{uid:" + UID + "}")); EXPECT(grid1->size() == SHAPE[0]); EXPECT(grid1->uid() == UID); std::unique_ptr grid2(GridFactory::build(spec::Custom({{"uid", UID}}))); EXPECT(grid2->size() == SHAPE[0]); EXPECT(grid2->uid() == UID); grid::unstructured::ICON grid3(UID); const std::string expected_spec_str = R"({"grid":")" + GRID + R"("})"; Log::info() << "'" << static_cast(grid3).spec_str() << "'" << std::endl; EXPECT(grid3.uid() == UID); EXPECT(static_cast(grid3).spec_str() == expected_spec_str); EXPECT(grid1->spec_str() == grid2->spec_str()); std::unique_ptr grid4(GridFactory::build(spec::Custom({{"grid", GRID}}))); EXPECT(grid4->spec_str() == expected_spec_str); std::unique_ptr grid5(GridFactory::build(spec::Custom({{"uid", UID}}))); EXPECT(*grid4 == *grid5); } CASE("equals") { std::unique_ptr grid1(GridFactory::make_from_string("{uid:" + UID + "}")); std::unique_ptr grid2(GridFactory::build(spec::Custom({{"uid", UID}}))); std::unique_ptr grid3(GridFactory::build(spec::Custom({{"grid", GRID}}))); grid::unstructured::ICON grid4(UID); EXPECT(*grid1 == *grid2); EXPECT(*grid2 == *grid3); EXPECT(*grid3 == grid4); EXPECT(grid4 == *grid1); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/gridspec.cc0000664000175000017500000001276715161702250017042 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/Grid.h" #include "eckit/geo/util.h" #include "eckit/log/Log.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("canonical") { for (const std::string& gridSpec : { R"({"area":[73,-27,33,45],"grid":[4,4]})", }) { std::unique_ptr grid(GridFactory::make_from_string(gridSpec)); EXPECT(grid); EXPECT(grid->spec_str() == gridSpec); } } CASE("user -> type") { using v = std::vector; static const std::string BAD; ASSERT(BAD.empty()); static std::pair tests[]{ {{{"N", 2}}, "reduced_gg"}, {{{"area", v{90, -180, -90, 180}}, {"grid", v{2, 2}}}, "regular_ll"}, {{{"area", v{90, -180, -90, 180}}}, BAD}, {{{"grid", "B48"}}, BAD}, {{{"grid", "F48"}}, "regular_gg"}, {{{"grid", "N48"}}, "reduced_gg"}, {{{"grid", "O48"}}, "reduced_gg"}, {{{"grid", 48}}, BAD}, {{{"grid", v{2, 2}}}, "regular_ll"}, {{{"grid", v{2, 2}}}, "regular_ll"}, {{{"grid", "48"}}, BAD}, {{{"grid", "F048"}}, BAD}, {{{"grid", "N"}}, BAD}, {{{"grid", "N048"}}, BAD}, {{{"grid", "N48"}}, "reduced_gg"}, {{{"grid", "O048"}}, BAD}, {{{"grid", "O48"}}, "reduced_gg"}, {{{"type", "reduced_gg"}}, ""}, {{{"grid", 2}}, ""}, {{{"grid", 12}}, ""}, {{{"type", "regular_gg"}, {"grid", "48"}}, BAD}, {{{"type", "regular_gg"}, {"grid", "F048"}}, BAD}, {{{"type", "regular_gg"}, {"grid", "F48"}}, "regular_gg"}, {{{"type", "regular_gg"}, {"grid", "N48"}}, "reduced_gg"}, {{{"type", "regular_gg"}, {"grid", "O48"}}, "reduced_gg"}, {{{"type", "regular_gg"}, {"grid", "a"}}, BAD}, {{{"type", "regular_gg"}, {"grid", 48}}, BAD}, {{{"type", "regular_ll"}, {"area", v{90, -180, -90, 180}}}, BAD}, {{{"type", "regular_ll"}, {"grid", "F48"}}, BAD}, {{{"type", "regular_ll"}, {"grid", "a"}}, BAD}, {{{"type", "regular_ll"}, {"grid", std::vector{"a", "b"}}}, BAD}, {{{"type", "regular_ll"}, {"grid", v{1, 2, 3}}}, BAD}, {{{"type", "regular_ll"}, {"grid", v{1, 2}}}, "regular_ll"}, {{{"type", "regular_ll"}, {"grid", v{1}}}, BAD}, {{{"type", "mercator"}, {"area", v{31.173058, 262.036499, 14.736453, 284.975281}}, {"grid", v{45000.0, 45000.0}}, {"shape", std::vector{56, 44}}, {"lad", 14.0}, {"orientation", 0.0}}, BAD}, }; for (const auto& [user, ref] : tests) { spec::Custom userspec(user); Log::info() << userspec << " -> type: " << ref << std::endl; try { std::unique_ptr spec(GridFactory::make_spec(userspec)); EXPECT(spec); std::unique_ptr grid(GridFactory::build(*spec)); EXPECT(grid); } catch (const exception::SpecError& e) { EXPECT(ref == BAD); } catch (const BadParameter& e) { EXPECT(ref == BAD); } } } CASE("grid: name -> spec -> grid: name") { // FIXME #if 0 for (const std::string& name : {"LAEA-EFAS-5km", "SMUFF-OPERA-2km"}) { std::unique_ptr grid(GridFactory::build(spec::Custom({{"grid", name}}))); EXPECT(grid); auto gridspec = grid->spec_str(); EXPECT(gridspec == R"({"grid":")" + name + R"("})"); } #endif } CASE("grid: reduced_gg") { std::unique_ptr o16(GridFactory::build(spec::Custom({{"grid", "o16"}}))); EXPECT(o16->spec_str() == R"({"grid":"O16"})"); std::unique_ptr n16(GridFactory::build(spec::Custom({{"grid", "n16"}}))); EXPECT(n16->spec_str() == R"({"grid":"N16"})"); std::unique_ptr known_pl_1(GridFactory::build( spec::Custom({{"pl", pl_type{20, 27, 32, 40, 45, 48, 60, 60, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 60, 60, 48, 45, 40, 32, 27, 20}}}))); EXPECT(known_pl_1->spec_str() == R"({"grid":"N16"})"); std::unique_ptr known_pl_2( GridFactory::build(spec::Custom({{"pl", pl_type{20, 24, 28, 32, 32, 28, 24, 20}}}))); EXPECT(known_pl_2->spec_str() == R"({"grid":"O4"})"); std::unique_ptr unknown_pl( GridFactory::build(spec::Custom({{"pl", pl_type{20, 24, 28, 32, 32, 28, 24, 99}}}))); EXPECT(unknown_pl->spec_str() == R"({"grid":"N4","pl":[20,24,28,32,32,28,24,99]})"); } CASE("grid: HEALPix") { std::unique_ptr h2(GridFactory::build(spec::Custom({{"grid", "h2"}}))); EXPECT(h2->spec_str() == R"({"grid":"H2"})"); std::unique_ptr h2n(GridFactory::build(spec::Custom({{"grid", "H2"}, {"order", "nested"}}))); EXPECT(h2n->spec_str() == R"({"grid":"H2","order":"nested"})"); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/projection_rotation.cc0000664000175000017500000002410015161702250021315 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/geo/area/BoundingBox.h" #include "eckit/geo/projection/Composer.h" #include "eckit/geo/projection/Rotation.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { using P = std::unique_ptr; constexpr double EPS = 1e-6; CASE("rotation (1)") { spec::Custom spec({ {"type", "rotation"}, {"south_pole_lat", -91.}, {"south_pole_lon", -361.}, }); Point p = PointLonLat{1, 1}; P projection(ProjectionFactoryType::instance().get(spec.get_string("type")).create(spec)); EXPECT(points_equal(p, projection->inv(projection->fwd(p)))); EXPECT(points_equal(p, projection->fwd(projection->inv(p)))); } CASE("rotation (2)") { const PointLonLat p(1, 1); int delta[] = {-360, -180, -1, 0, 1, 90, 91, 180}; for (auto a : delta) { for (auto b : delta) { for (auto c : delta) { projection::Rotation rot({0. + static_cast(b), -90. + static_cast(a)}, static_cast(c)); EXPECT(rot.rotated() == (a % 360 != 0 || (b - c) % 360 != 0)); EXPECT(points_equal(p, rot.inv(rot.fwd(p)), EPS)); EXPECT(points_equal(p, rot.fwd(rot.inv(p)), EPS)); } } } } CASE("rotation (3)") { const PointLonLat sp(182., -46.7); projection::Rotation rot({sp.lon(), sp.lat()}, 0.); ASSERT(points_equal(sp.antipode(), PointLonLat{2., 46.7})); SECTION("pole point") { PointLonLat a(0., 90.); auto b = rot.fwd(a); auto c = rot.inv(b); EXPECT(points_equal(b, sp.antipode(), EPS)); EXPECT(points_equal(a, c, EPS)); } SECTION("many points") { const int Ni = 12; const int Nj = 3; const PointLonLat ref[]{ {-178., -46.7}, // {-178., -16.7}, {-178., 13.3}, {-178., 43.3}, {-178., 73.3}, {2., 76.7}, {2., 46.7}, {-178., -46.7}, {-162.623427, -19.469294}, {-152.023657, 8.654593}, {-139.574639, 36.436827}, {-113.108943, 61.431994}, {-39.882454, 68.008245}, {2., 46.7}, {-178., -46.7}, {-148.834426, -27.310675}, {-129.263456, -3.837005}, {-110.791162, 20.054216}, {-85.879167, 41.365070}, {-44.424956, 53.295076}, {2., 46.7}, {-178., -46.7}, {-137.907940, -39.070023}, {-109.601456, -21.339065}, {-88., 0.}, {-66.398544, 21.339065}, {-38.092060, 39.070023}, {2., 46.7}, {-178., -46.7}, {-131.575044, -53.295076}, {-90.120833, -41.365070}, {-65.208838, -20.054216}, {-46.736544, 3.837005}, {-27.165574, 27.310675}, {2., 46.7}, {-178., -46.7}, {-136.117546, -68.008245}, {-62.891057, -61.431994}, {-36.425361, -36.436827}, {-23.976343, -8.654593}, {-13.376573, 19.469294}, {2., 46.7}, {-178., -46.7}, {-178., -76.7}, {2., -73.3}, {2., -43.3}, {2., -13.3}, {2., 16.7}, {2., 46.7}, {-178., -46.7}, {140.117546, -68.008245}, {66.891057, -61.431994}, {40.425361, -36.436827}, {27.976343, -8.654593}, {17.376573, 19.469294}, {2., 46.7}, {-178., -46.7}, {135.575044, -53.295076}, {94.120833, -41.365070}, {69.208838, -20.054216}, {50.736544, 3.837005}, {31.165574, 27.310675}, {2., 46.7}, {-178., -46.7}, {141.907940, -39.070023}, {113.601456, -21.339065}, {92., 0.}, {70.398544, 21.339065}, {42.092060, 39.070023}, {2., 46.7}, {-178., -46.7}, {152.834426, -27.310675}, {133.263456, -3.837005}, {114.791162, 20.054216}, {89.879167, 41.365070}, {48.424956, 53.295076}, {2., 46.7}, {-178., -46.7}, {166.623427, -19.469294}, {156.023657, 8.654593}, {143.574639, 36.436827}, {117.108943, 61.431994}, {43.882454, 68.008245}, {2., 46.7}, }; for (int i = 0, k = 0; i < Ni; i++) { for (int j = 0; j < 2 * Nj + 1; j++, k++) { PointLonLat a(static_cast(i) * 360. / static_cast(Ni), static_cast(j - Nj) * 90. / static_cast(Nj)); auto b = rot.fwd(a); auto c = rot.inv(b); EXPECT(points_equal(b, ref[k], EPS)); EXPECT(points_equal(a, c, EPS)); } } } } CASE("rotation (4)") { const projection::Rotation non_rotated({0., -90.}, 0.); const projection::Rotation rotation_angle({0., -90.}, -180.); const projection::Rotation rotation_matrix({4., -40.}, 180.); EXPECT(not non_rotated.rotated()); EXPECT(rotation_angle.rotated()); EXPECT(rotation_matrix.rotated()); const PointLonLat p[] = {{0., 90.}, {0., 0.}, {270., 25.}, {-180., 45.}}; struct { const projection::Rotation& rotation; const PointLonLat a; const PointLonLat b; } tests[] = { {non_rotated, p[0], p[0]}, {non_rotated, p[1], p[1]}, {non_rotated, p[2], p[2]}, {non_rotated, p[3], p[3]}, {rotation_angle, p[0], {p[0].lon() - 180., p[0].lat()}}, {rotation_angle, p[1], {p[1].lon() - 180., p[1].lat()}}, {rotation_angle, p[2], {p[2].lon() - 180., p[2].lat()}}, {rotation_angle, p[3], {p[3].lon() - 180., p[3].lat()}}, {rotation_matrix, p[0], {-176., 40.}}, {rotation_matrix, p[1], {-176., -50.}}, {rotation_matrix, p[2], {113.657357, 15.762700}}, {rotation_matrix, p[3], {-176., 85.}}, }; for (const auto& test : tests) { auto b = test.rotation.fwd(test.a); EXPECT(points_equal(b, test.b, EPS)); auto a = test.rotation.inv(b); EXPECT(points_equal(a, test.a, EPS)); } } CASE("rotation (5)") { spec::Custom spec({ {"type", "rotation"}, {"south_pole_lat", -90.}, {"south_pole_lon", 0.}, {"rotation_angle", 45.}, }); // compose sequentially const auto& builder = ProjectionFactoryType::instance().get(spec.get_string("type")); P composition1(new projection::Composer{ builder.create(spec), builder.create(spec), builder.create(spec), builder.create(spec), builder.create(spec), builder.create(spec), builder.create(spec), }); dynamic_cast(composition1.get())->emplace_back(builder.create(spec)); for (auto lat : {0., 10., -10.}) { PointLonLat p{0., lat}; auto q = composition1->fwd(p); EXPECT(points_equal(p, q)); EXPECT(points_equal(p, composition1->inv(q))); auto qs = dynamic_cast(composition1.get())->fwd_points(p); EXPECT(qs.size() == 8); EXPECT(points_equal(qs.front(), PointLonLat{-45., lat})); EXPECT(points_equal(qs[1], PointLonLat{-90., lat})); EXPECT(points_equal(qs[2], PointLonLat{-135., lat})); // ... EXPECT(points_equal(qs.back(), p)); } // compose by nesting P composition2(builder.create(spec)); for (size_t i = 1; i < 8; ++i) { composition2.reset(projection::Composer::compose_back(composition2.release(), spec)); } for (auto lat : {0., 10., -10.}) { PointLonLat p{0., lat}; auto qs1 = dynamic_cast(composition1.get())->fwd_points(p); auto qs2 = dynamic_cast(composition2.get())->fwd_points(p); ASSERT(qs1.size() == 8); EXPECT(qs2.size() == 2); EXPECT(points_equal(qs1[6], qs2[0])); EXPECT(points_equal(qs1[7], qs2[1])); } } CASE("make_from_projection (1)") { projection::Rotation rotation(spec::Custom({ {"type", "rotation"}, {"south_pole_lat", 10.}, {"south_pole_lon", 20.}, })); struct test_t { PointLonLat p; PointLonLat q; }; for (const auto& test : std::vector{ {{0., 1.}, {-160., 79.}}, {{2., 1.}, {-170.369068, 78.821318}}, {{0., 0.}, {-160., 80.}}, {{2., 0.}, {-171.370559, 79.803957}}, }) { EXPECT(points_equal(rotation.fwd(test.p), test.q, EPS)); } EXPECT_THROWS_AS(auto dummy = area::BoundingBox::make_from_projection(PointXY{0., 0.}, PointXY{2., 1.}, rotation), std::bad_variant_access); auto bbox = area::BoundingBox::make_from_projection(PointLonLat{0., 0.}, PointLonLat{2., 1.}, rotation); ASSERT(bbox); EXPECT(points_equal(PointLonLat{bbox->west(), bbox->north()}, {-171.37056, 80.000001}, EPS)); EXPECT(points_equal(PointLonLat{bbox->east(), bbox->south()}, {-160., 78.821318}, EPS)); } CASE("make_from_projection (2)") { projection::Rotation rotation({22, -40}); PointLonLat min(-27, 33); PointLonLat max(45, 73); auto bbox = area::BoundingBox::make_from_projection(min, max, rotation); ASSERT(bbox); EXPECT(points_equal(PointLonLat{0, bbox->north()}, NORTH_POLE)); EXPECT(bbox->periodic()); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/grid_orca.cc0000664000175000017500000000667615161702250017175 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to itr by virtue of its status as an intergovernmental organisation nor * does itr submit to any jurisdiction. */ #include #include #include #include "eckit/geo/LibEcKitGeo.h" #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/grid/ORCA.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { static const std::string GRID = "ORCA2_T"; static const Grid::uid_type UID = "d5bde4f52ff3a9bea5629cd9ac514410"; static const std::vector SHAPE{182, 149}; CASE("caching") { if (LibEcKitGeo::caching()) { using Cache = cache::MemoryCache; SECTION("Grid::build_from_uid") { spec::Custom spec({{"uid", UID}}); const auto footprint_1 = Cache::total_footprint(); std::unique_ptr grid1(GridFactory::build(spec)); const auto footprint_2 = Cache::total_footprint(); EXPECT(footprint_1 < footprint_2); std::unique_ptr grid2(GridFactory::build(spec)); EXPECT(footprint_2 == Cache::total_footprint()); EXPECT(grid1->size() == grid2->size()); } } } CASE("spec") { std::unique_ptr spec(GridFactory::make_spec(spec::Custom({{"uid", UID}}))); EXPECT(spec->get_string("type") == "ORCA"); EXPECT(spec->get_string("name") == "ORCA2"); EXPECT(spec->get_string("orca_arrangement") == "T"); EXPECT(spec->get_string("orca_uid") == UID); EXPECT(spec->get_long_vector("dimensions") == SHAPE); std::unique_ptr grid1(GridFactory::make_from_string("{uid:" + UID + "}")); EXPECT(grid1->size() == SHAPE[0] * SHAPE[1]); EXPECT(grid1->uid() == UID); std::unique_ptr grid2(GridFactory::build(spec::Custom({{"uid", UID}}))); EXPECT(grid2->size() == SHAPE[0] * SHAPE[1]); EXPECT(grid2->uid() == UID); grid::ORCA grid3(UID); const std::string expected_spec_str = R"({"grid":")" + GRID + R"("})"; Log::info() << "'" << static_cast(grid3).spec_str() << "'" << std::endl; EXPECT(grid3.uid() == UID); EXPECT(grid3.calculate_uid() == UID); EXPECT(static_cast(grid3).spec_str() == expected_spec_str); EXPECT(grid1->spec_str() == grid2->spec_str()); std::unique_ptr grid4(GridFactory::build(spec::Custom({{"grid", GRID}}))); EXPECT(grid4->spec_str() == expected_spec_str); std::unique_ptr grid5(GridFactory::build(spec::Custom({{"uid", UID}}))); EXPECT(*grid4 == *grid5); } CASE("equals") { std::unique_ptr grid1(GridFactory::make_from_string("{uid:" + UID + "}")); std::unique_ptr grid2(GridFactory::build(spec::Custom({{"uid", UID}}))); std::unique_ptr grid3(GridFactory::build(spec::Custom({{"grid", GRID}}))); grid::ORCA grid4(UID); EXPECT(*grid1 == *grid2); EXPECT(*grid2 == *grid3); EXPECT(*grid3 == grid4); EXPECT(grid4 == *grid1); EXPECT(grid2->uid() == grid3->calculate_uid()); EXPECT(grid3->uid() == grid2->calculate_uid()); } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/projection_albers.cc0000664000175000017500000000421215161702250020730 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/eckit_geo_config.h" #include "eckit/geo/figure/Sphere.h" #include "eckit/geo/projection/AlbersEqualArea.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("AlbersEqualArea") { #if eckit_HAVE_PROJ constexpr auto eps = 10. * PointXY::EPS; // FIXME improve floating-point errors projection::AlbersEqualArea proj1(-96., 23., 29.5, 45.5, new figure::Sphere(6378206.4)); std::unique_ptr proj2( ProjectionFactory::has_type("proj") ? ProjectionFactory::build(spec::Custom{{"type", "proj"}, {"proj", proj1.proj_str()}}) : nullptr); SECTION("proj_str") { EXPECT(proj1.proj_str() == R"(+proj=aea +lat_0=23 +lat_1=29.5 +lat_2=45.5 +lon_0=-96 +R=6378206.4)"); } SECTION("inv(fwd(.)) == . and fwd(inv(.)) == .") { auto a = PointXY{0., 0.}; auto aa = proj1.inv(a); auto aaa = proj1.fwd(aa); EXPECT(points_equal(a, aaa, eps)); auto b = PointLonLat{-75., 35.}; auto bb = proj1.fwd(b); auto bbb = proj1.inv(bb); EXPECT(points_equal(b, bbb)); if (proj2) { auto aaaa = std::get(proj2->inv(a)); auto aaaaa = std::get(proj2->fwd(aaaa)); EXPECT(points_equal(aa, aaaa)); EXPECT(points_equal(a, aaaaa, eps)); auto bbbb = std::get(proj2->fwd(b)); auto bbbbb = std::get(proj2->inv(bbbb)); // EXPECT(points_equal(bb, bbbb, eps)); EXPECT(points_equal(b, bbbbb)); } } #endif } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/grid_to_points.cc0000664000175000017500000000523115161702250020251 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/geo/grid/reduced/HEALPix.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("HEALPix") { SECTION("HEALPix::to_points") { std::unique_ptr grid(new grid::reduced::HEALPix(2, order::HEALPix::RING)); static const std::vector expected_points_ring{ {45, 66.443535691}, {135, 66.443535691}, {225, 66.443535691}, {315, 66.443535691}, {22.5, 41.810314896}, {67.5, 41.810314896}, {112.5, 41.810314896}, {157.5, 41.810314896}, {202.5, 41.810314896}, {247.5, 41.810314896}, {292.5, 41.810314896}, {337.5, 41.810314896}, {0, 19.471220634}, {45, 19.471220634}, {90, 19.471220634}, {135, 19.471220634}, {180, 19.471220634}, {225, 19.471220634}, {270, 19.471220634}, {315, 19.471220634}, {22.5, 0}, {67.5, 0}, {112.5, 0}, {157.5, 0}, {202.5, 0}, {247.5, 0}, {292.5, 0}, {337.5, 0}, {0, -19.471220634}, {45, -19.471220634}, {90, -19.471220634}, {135, -19.471220634}, {180, -19.471220634}, {225, -19.471220634}, {270, -19.471220634}, {315, -19.471220634}, {22.5, -41.810314896}, {67.5, -41.810314896}, {112.5, -41.810314896}, {157.5, -41.810314896}, {202.5, -41.810314896}, {247.5, -41.810314896}, {292.5, -41.810314896}, {337.5, -41.810314896}, {45, -66.443535691}, {135, -66.443535691}, {225, -66.443535691}, {315, -66.443535691}, }; auto points = grid->to_points(); EXPECT_EQUAL(points.size(), expected_points_ring.size()); for (int i = 0; i < points.size(); ++i) { EXPECT(points_equal(std::get(points[i]), expected_points_ring[i])); } } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/grid_regular_ll.cc0000664000175000017500000001135115161702250020363 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/grid/regular/RegularLL.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { using grid::regular::RegularLL; CASE("global") { SECTION("1") { std::unique_ptr grid(new RegularLL(spec::Custom{{{"grid", std::vector{90, 90}}}})); EXPECT(grid->size() == 4 * 3); std::vector ref{{0., 90.}, {90., 90.}, {180., 90.}, {270., 90.}, // {0., 0.}, {90., 0.}, {180., 0.}, {270., 0.}, // {0., -90.}, {90., -90.}, {180., -90.}, {270., -90.}}; EXPECT(points_equal(grid->first_point(), ref.front())); EXPECT(points_equal(grid->last_point(), ref.back())); const auto points = grid->to_points(); ASSERT(points.size() == ref.size()); EXPECT(points.size() == grid->size()); auto q = ref.cbegin(); for (const auto& p : points) { EXPECT(points_equal(p, *q++)); } } SECTION("2") { RegularLL a(spec::Custom{{{"grid", std::vector{1, 1}}}}); EXPECT(a.size() == 360 * 181); EXPECT(a.spec_str() == R"({"grid":[1,1]})"); RegularLL b(spec::Custom{{{"grid", std::vector{2, 1}}, {"area", std::vector{10, 1, 1, 10}}}}); EXPECT(b.size() == 5 * 10); EXPECT(b.spec_str() == R"({"area":[10,1,1,9],"grid":[2,1]})"); RegularLL c({1., 1.}, {89.5, 0.5, -89.5, 359.5}, {0.5, 0.5}); EXPECT(c.nlon() == 360); EXPECT(c.nlat() == 180); EXPECT(c.size() == 360 * 180); EXPECT(c.spec_str() == R"({"area":[90,0.5,-90,360.5],"grid":[1,1]})"); RegularLL d({1., 1.}, {90., 0.5, -90, 360.5}, {0.5, 0.5}); EXPECT(d.nlon() == 360); EXPECT(d.nlat() == 180); EXPECT(d.size() == 360 * 180); } SECTION("3") { for (const double d : { 7., 0.35, 0.7, 0.8, 1.4, 1.6, }) { const auto spec_ref = RegularLL({d, d}).spec_str(); std::unique_ptr grid1(GridFactory::build(spec::Custom{{{"grid", std::vector{d, d}}}})); EXPECT(spec_ref == grid1->spec_str()); std::unique_ptr grid2( GridFactory::make_from_string(std::to_string(d) + "/" + std::to_string(d))); EXPECT(spec_ref == grid2->spec_str()); } } } CASE("non-global") { SECTION("origin at (0, 0) (non-shifted)") { /* * 1 * 0 . . . . * -1 * -1 0 1 2 */ RegularLL grid({1, 2}, {1, -1, -1, 2}); const std::vector ref{{-1., 0.}, {0., 0.}, {1., 0.}, {2., 0.}}; auto points = grid.to_points(); ASSERT(points.size() == ref.size()); EXPECT(points.size() == grid.size()); auto it = grid.begin(); for (size_t i = 0; i < points.size(); ++i) { EXPECT(points_equal(ref[i], points[i])); EXPECT(points_equal(ref[i], *it)); ++it; } EXPECT(it == grid.end()); size_t i = 0; for (const auto& it : grid) { EXPECT(points_equal(ref[i++], it)); } EXPECT(i == grid.size()); } SECTION("origin at (-1, -1) (shifted)") { /* * 1 . . . . * 0 * -1 . . . . * -1 0 1 2 */ RegularLL grid({1, 2}, {1, -1, -1, 2}, {-1, -1}); const std::vector ref{ {-1., 1.}, {0., 1.}, {1., 1.}, {2., 1.}, {-1., -1.}, {0., -1.}, {1., -1.}, {2., -1.}, }; auto points = grid.to_points(); ASSERT(points.size() == ref.size()); EXPECT(points.size() == grid.size()); auto it = grid.begin(); for (size_t i = 0; i < points.size(); ++i) { EXPECT(points_equal(ref[i], points[i])); EXPECT(points_equal(ref[i], *it)); ++it; } EXPECT(it == grid.end()); size_t i = 0; for (const auto& it : grid) { EXPECT(points_equal(ref[i++], it)); } EXPECT(i == grid.size()); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/figure_earth.cc0000664000175000017500000001012015161702250017663 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/geo/figure/Earth.h" #include "eckit/geo/projection/LonLatToXYZ.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::test { struct F : std::shared_ptr
{ explicit F(Figure* ptr) : shared_ptr(ptr) { ASSERT(operator bool()); } }; CASE("Earth") { F f1(FigureFactory::build(spec::Custom{{"r", 6371229.}})); F f2(new figure::Earth); EXPECT(*f1 == *f2); EXPECT(f1->spec_str() == R"({"r":6371229})"); EXPECT(types::is_approximately_equal(f1->R(), 6371229., 1e-8)); F f4(FigureFactory::build(spec::Custom{{"figure", "wgs84"}})); EXPECT(f4->spec_str() == R"({"figure":"wgs84"})"); EXPECT(types::is_approximately_equal(1. / f4->flattening(), 298.257223563, 1e-8)); EXPECT_THROWS_AS(f4->R(), BadValue); } CASE("Area") { struct test_t { test_t(const std::string& _figure, double _area) : figure(FigureFactory::build(spec::Custom{{"figure", _figure}})), area(_area) {} const F figure; const double area; }; for (const auto& test : std::vector{ {"earth", 510101140.}, //[km^2] {"wgs84", 510065621.}, {"grs80", 510065621.}, }) { EXPECT(types::is_approximately_equal(test.figure->area() * 1e-6 /*[km^2]*/, test.area, 1.)); } } CASE("projection::LonLatToXYZ") { const auto R = figure::EARTH.R(); const auto L = R * std::sqrt(2) / 2.; projection::LonLatToXYZ to_xyz(new figure::Earth); projection::LonLatToXYZ to_xyz_default; SECTION("lon 0 (quadrant)") { PointXYZ p{R, 0, 0}; EXPECT(points_equal(to_xyz.fwd({0., 0.}), p)); EXPECT(points_equal(to_xyz.fwd({-360., 0.}), p)); EXPECT(points_equal(to_xyz_default.fwd({0., 0.}), p)); } SECTION("lon 90 (quadrant)") { PointXYZ p{0, R, 0}; EXPECT(points_equal(to_xyz.fwd({90., 0.}), p)); EXPECT(points_equal(to_xyz.fwd({-270., 0.}), p)); EXPECT(points_equal(to_xyz_default.fwd({90., 0.}), p)); } SECTION("lon 180 (quadrant)") { PointXYZ p{-R, 0, 0}; EXPECT(points_equal(to_xyz.fwd({180., 0.}), p)); EXPECT(points_equal(to_xyz.fwd({-180., 0.}), p)); EXPECT(points_equal(to_xyz_default.fwd({180., 0.}), p)); } SECTION("lon 270 (quadrant)") { PointXYZ p{0, -R, 0}; EXPECT(points_equal(to_xyz.fwd({270., 0.}), p)); EXPECT(points_equal(to_xyz.fwd({-90., 0.}), p)); EXPECT(points_equal(to_xyz_default.fwd({270., 0.}), p)); } SECTION("lon 45 (octant)") { PointXYZ p{L, L, 0}; EXPECT(points_equal(to_xyz.fwd({45., 0.}), p)); EXPECT(points_equal(to_xyz.fwd({-315., 0.}), p)); EXPECT(points_equal(to_xyz_default.fwd({45., 0.}), p)); } SECTION("lon 135 (octant)") { PointXYZ p{-L, L, 0}; EXPECT(points_equal(to_xyz.fwd({135., 0.}), p)); EXPECT(points_equal(to_xyz.fwd({-225., 0.}), p)); EXPECT(points_equal(to_xyz_default.fwd({135., 0.}), p)); } SECTION("lon 225 (octant)") { PointXYZ p{-L, -L, 0}; EXPECT(points_equal(to_xyz.fwd({225., 0.}), p)); EXPECT(points_equal(to_xyz.fwd({-135., 0.}), p)); EXPECT(points_equal(to_xyz_default.fwd({225., 0.}), p)); } SECTION("lon 315 (octant)") { PointXYZ p{L, -L, 0}; EXPECT(points_equal(to_xyz.fwd({315., 0.}), p)); EXPECT(points_equal(to_xyz.fwd({-45., 0.}), p)); EXPECT(points_equal(to_xyz_default.fwd({315., 0.}), p)); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/CNTR_RG_60M_2024_4326_mini.geojson0000664000175000017500000002574615161702250022243 0ustar alastairalastair{"type":"FeatureCollection","name":"CNTR_RG_60M_2024_4326.geojson","crs":{"type":"name","properties":{"name":"urn:ogc:def:crs:EPSG::4326"}},"features":[{"type":"Feature","properties":{"CNTR_ID":"ES","CNTR_NAME":"España","NAME_ENGL":"Spain","NAME_FREN":"Espagne","ISO3_CODE":"ESP","SVRG_UN":"UN Member State","CAPT":"Madrid","EU_STAT":"T","EFTA_STAT":"F","CC_STAT":"F","NAME_GERM":"Spanien"},"geometry":{"type":"MultiPolygon","coordinates":[[[[4.316571345000057,39.876461059000064],[4.200062758000058,39.83676379000008],[3.839899983000066,39.92628637100006],[3.767017022000061,40.02187913800003],[4.160211701000037,40.060846172000026],[4.316571345000057,39.876461059000064]]],[[[3.339049795000051,39.672256977000075],[2.986364133000052,39.32426871300004],[2.374801400000024,39.53513728100006],[2.95103969500002,39.89826164100003],[3.339049795000051,39.672256977000075]]],[[[-7.699736359999974,43.735114829000054],[-7.031836889999965,43.54447142200007],[-5.837651206999965,43.610603764000075],[-4.512301089999937,43.39320382100004],[-3.582614975999945,43.51192161100005],[-3.153338109999936,43.35322211300007],[-2.41284679499995,43.32108296600006],[-1.818383367999957,43.34905943500007],[-1.785981393999975,43.350584328000025],[-1.728903005999939,43.29608895400003],[-0.724501161999967,42.92015867100008],[-0.313341956999977,42.849364937000075],[-0.107639497999969,42.81053495200007],[0.477670138000065,42.70004738500006],[0.660127054000043,42.69095264900005],[0.858215053000038,42.82574115800003],[1.235239801000034,42.68853978900006],[1.429658890000042,42.617789619000064],[1.460137365000037,42.606119954000064],[1.725801230000059,42.50440199700006],[1.731010891000039,42.49240083700005],[2.154506831000049,42.42256387600003],[2.970126917000073,42.432704183000055],[3.174022446000038,42.43523914200006],[3.173376363000045,42.43358578300007],[3.167380557000058,42.31709342400006],[3.145744674000071,41.89673044300008],[2.970126917000073,41.778168685000026],[2.778512932000069,41.64880767200003],[1.645323423000036,41.19562168500005],[1.460137365000037,41.14243574200003],[1.235239801000034,41.07784454600005],[0.946872615000075,40.995024701000034],[0.515238434000025,40.522918609000044],[-0.107639497999969,39.81906444700007],[-0.170245038999951,39.748319974000026],[-0.284252964999951,39.43463793300003],[-0.107639497999969,39.042061674000024],[-0.037615968999944,38.886413436000055],[0.137161644000059,38.71599020900004],[-0.107639497999969,38.55651444500006],[-0.495465976999981,38.303864789000045],[-0.762135483999941,37.84700783900007],[-0.821050622999962,37.70023374600004],[-0.928294817999927,37.58736782500006],[-1.629967653999927,37.37515068600004],[-1.818383367999957,37.12260996500004],[-1.99302487999995,36.888531328000056],[-2.193818140999952,36.76321296300006],[-2.416063311999949,36.76028279800005],[-3.128676713999937,36.75088743200007],[-3.582614975999945,36.72586498000004],[-3.777457583999933,36.73792632300007],[-4.34430929399997,36.70704537300003],[-5.252405121999971,36.31127557600007],[-5.339031880999926,36.15336726400005],[-5.33922483799995,36.152035074000025],[-5.351599546999978,36.15259099800005],[-5.351427162999926,36.153360629000076],[-5.611857446999977,36.044275527000025],[-6.003119865999963,36.191155147000075],[-6.347371774999942,36.60204238800003],[-6.34556101499993,36.79876695000007],[-6.917095128999961,37.149549464000074],[-7.401916633999974,37.17482748000003],[-7.512691485999937,37.52625636400006],[-6.931738572999961,38.208378046000064],[-7.107952585999953,38.188121617000036],[-7.27497881499994,38.443379883000034],[-7.203134912999928,38.75101749100003],[-7.019914536999977,39.00353529500006],[-7.231467159999966,39.278431039000054],[-7.542845391999947,39.66278609400007],[-7.010466545999975,39.73727061000005],[-6.951298650999945,40.257446025000036],[-6.865144205999968,40.27069454400004],[-6.929864671999951,41.02933623200005],[-6.689786233999939,41.20524137500007],[-6.479713167999932,41.29437983400004],[-6.261031507999974,41.552111350000075],[-6.570356939999954,41.884722543000066],[-6.985800483999981,41.971447057000034],[-7.20046440599998,41.87974942100004],[-8.051862724999978,41.82061392600008],[-8.165075344999934,41.81830214400003],[-8.19900032399994,42.15441906400008],[-8.307231525999953,42.10840878400006],[-8.863185838999925,41.87206646300007],[-8.769266196999979,42.10840878400006],[-8.693736982999951,42.29847285900007],[-8.821788653999931,42.511704233000046],[-8.72674928999993,42.68825152200003],[-8.992542407999963,42.650103632000025],[-9.057485898999971,42.75726691700004],[-9.23967691799993,43.057900406000044],[-8.631949341999928,43.31084115500005],[-7.699736359999974,43.735114829000054]]],[[[1.608049816000062,39.05345643600003],[1.420727589000023,38.81600497000005],[1.11903952800003,38.96619097000007],[1.410565359000032,39.11122953900008],[1.608049816000062,39.05345643600003]]],[[[-2.438735909999934,35.17807307000004],[-2.440229236999926,35.17688099800006],[-2.444070082999929,35.178685478000034],[-2.441351853999947,35.18317855500004],[-2.439973238999926,35.18324332800006],[-2.438735909999934,35.17807307000004]]],[[[-2.929703441999948,35.271704476000025],[-2.949435280999978,35.26624666500004],[-2.967586141999959,35.282838679000065],[-2.960128251999947,35.31518301800003],[-2.952482233999945,35.31990132300007],[-2.946098933999963,35.31429909700006],[-2.945482495999954,35.30752620000004],[-2.932295653999972,35.29356603400004],[-2.935818074999929,35.29130747100004],[-2.93709492499994,35.28313634600005],[-2.929703441999948,35.271704476000025]]],[[[-5.279957535999927,35.894948025000076],[-5.344183880999935,35.871149659000025],[-5.382030037999925,35.91260328800007],[-5.37807825799996,35.91634251200003],[-5.36619785399995,35.91696062900007],[-5.360083919999965,35.913756686000056],[-5.336582084999975,35.895372991000045],[-5.279957535999927,35.894948025000076]]],[[[-13.37379954399995,29.134539579000034],[-13.58177563199996,28.900067027000034],[-13.878392774999952,28.98439475500004],[-13.642488594999975,29.16986637800005],[-13.37379954399995,29.134539579000034]]],[[[-13.899169464999943,28.262220831000036],[-14.082216204999952,28.155286593000028],[-14.255763297999977,28.198391910000055],[-14.00569060099997,28.71007452200007],[-13.844534150999948,28.746488682000063],[-13.899169464999943,28.262220831000036]]],[[[-15.333067459999938,27.94400032200008],[-15.573544659999982,27.716246054000067],[-15.923425691999967,27.975731693000057],[-15.547177205999958,28.199528057000066],[-15.333067459999938,27.94400032200008]]],[[[-16.32511081299998,28.437807856000063],[-16.60807159999996,28.00939211900004],[-16.78415057199993,28.076247141000067],[-16.987127038999972,28.363388327000052],[-16.43655744499995,28.504742436000072],[-16.32511081299998,28.437807856000063]]],[[[-17.014797577999957,28.08358183400003],[-17.232546879999973,27.963013146000037],[-17.481537646999982,28.083547940000074],[-17.268933482999955,28.274221545000046],[-17.014797577999957,28.08358183400003]]],[[[-17.721523572999956,28.83094310000007],[-17.777012183999943,28.489845657000046],[-18.071698864999973,28.802704337000023],[-17.903474594999977,28.894453076000048],[-17.721523572999956,28.83094310000007]]],[[[-17.800534066999944,27.880589233000023],[-17.968694924999966,27.576453769000068],[-18.251031057999967,27.71397749500005],[-17.944338672999947,27.90789372000006],[-17.800534066999944,27.880589233000023]]]]}},{"type":"Feature","properties":{"CNTR_ID":"PT","CNTR_NAME":"Portugal","NAME_ENGL":"Portugal","NAME_FREN":"Portugal","ISO3_CODE":"PRT","SVRG_UN":"UN Member State","CAPT":"Lisbon","EU_STAT":"T","EFTA_STAT":"F","CC_STAT":"F","NAME_GERM":"Portugal"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-8.19900032399994,42.15441906400008],[-8.165075344999934,41.81830214400003],[-8.051862724999978,41.82061392600008],[-7.20046440599998,41.87974942100004],[-6.985800483999981,41.971447057000034],[-6.570356939999954,41.884722543000066],[-6.261031507999974,41.552111350000075],[-6.479713167999932,41.29437983400004],[-6.689786233999939,41.20524137500007],[-6.929864671999951,41.02933623200005],[-6.865144205999968,40.27069454400004],[-6.951298650999945,40.257446025000036],[-7.010466545999975,39.73727061000005],[-7.542845391999947,39.66278609400007],[-7.231467159999966,39.278431039000054],[-7.019914536999977,39.00353529500006],[-7.203134912999928,38.75101749100003],[-7.27497881499994,38.443379883000034],[-7.107952585999953,38.188121617000036],[-6.931738572999961,38.208378046000064],[-7.512691485999937,37.52625636400006],[-7.401916633999974,37.17482748000003],[-7.401590665999947,37.174407967000036],[-7.820882664999942,37.03309372600006],[-8.82644189399997,37.089055478000034],[-8.796265455999958,37.44287863000005],[-8.735019809999926,38.51569646100006],[-9.01950114899995,38.563256498000044],[-8.924880813999948,38.758673875000056],[-8.96846585399993,38.827764847000026],[-9.406584205999934,38.72933333200007],[-9.416395121999926,39.05475934100008],[-9.040260312999976,39.741423033000046],[-8.894932676999929,40.04550315700004],[-8.78402916899995,40.520365051000056],[-8.653128771999945,40.964761520000025],[-8.776427954999974,41.47197131200005],[-8.811574118999943,41.61156754000007],[-8.871530104999977,41.86390114100004],[-8.863185838999925,41.87206646300007],[-8.307231525999953,42.10840878400006],[-8.19900032399994,42.15441906400008]]],[[[-15.819650916999933,30.02535143800003],[-16.013796635999938,29.859042419000048],[-16.245820953999953,30.05982689800004],[-16.021357895999927,30.20049143700004],[-15.819650916999933,30.02535143800003]]],[[[-16.582585834999975,32.771601721000025],[-16.812949747999937,32.60898334400002],[-17.13577199699995,32.73805116600005],[-16.821771858999966,32.87102655100006],[-16.582585834999975,32.771601721000025]]],[[[-25.175033480999957,37.747141620000036],[-25.373742516999982,37.71466527300004],[-25.522928108999963,37.71655323700003],[-25.571812545999933,37.771889015000056],[-25.498097247999965,37.83886980400007],[-25.28164456299993,37.86071424000005],[-25.166911454999934,37.851555759000064],[-25.14010075599998,37.805426940000075],[-25.175033480999957,37.747141620000036]]],[[[-25.58637608999993,37.74958427900003],[-25.767622664999976,37.68309650300006],[-26.020811143999936,37.89581237300007],[-25.72148994799994,38.00192152100004],[-25.58637608999993,37.74958427900003]]],[[[-26.834941063999963,38.75559296600005],[-27.07642991499995,38.58747171300007],[-27.386756799999944,38.696405782000056],[-27.062713563999978,38.85217328900006],[-26.834941063999963,38.75559296600005]]],[[[-28.326101016999928,38.41502582600003],[-28.424031759999934,38.41021891300005],[-28.578012480999973,38.51665321200005],[-28.503668129999937,38.55745728000005],[-28.396797961999937,38.563442259000055],[-28.306840190999935,38.52349376300003],[-28.28148693099996,38.46697007400007],[-28.326101016999928,38.41502582600003]]],[[[-28.502351116999932,38.56205371500005],[-28.695167413999968,38.46242261500004],[-28.914778402999957,38.58221646100003],[-28.664302868999982,38.686318252000035],[-28.502351116999932,38.56205371500005]]],[[[-31.019204526999943,39.44805220100005],[-31.219158169999957,39.25008026800003],[-31.404938543999947,39.470888080000066],[-31.226068555999973,39.627056270000026],[-31.019204526999943,39.44805220100005]]]]}}]} eckit-2.0.7/tests/geo/CMakeLists.txt0000664000175000017500000000370715161702250017465 0ustar alastairalastairforeach(_test area_boundingbox area_polygon cache figure figure_earth gaussian great_circle grid grid_healpix grid_reduced_gg grid_reduced_ll grid_regular_gg grid_regular_ll grid_reorder grid_to_points gridspec integration iterator kdtree order point point2 point3 pointlonlat pointlonlatr projection projection_albers projection_ll_to_xyz projection_mercator projection_plate-caree projection_rotation range search util ) ecbuild_add_test( TARGET eckit_test_geo_${_test} SOURCES ${_test}.cc LIBS eckit_geo ENVIRONMENT "ECKIT_GEO_CACHE_PATH=${CMAKE_CURRENT_SOURCE_DIR}/eckit_geo_cache" ) endforeach() set_tests_properties( eckit_test_geo_search PROPERTIES ENVIRONMENT "ECKIT_GEO_CACHE_PATH=${CMAKE_CURRENT_BINARY_DIR}/eckit_geo_cache" ) if(eckit_HAVE_ZIP) target_compile_definitions(eckit_test_geo_cache PRIVATE ZIP_FILE="${CMAKE_CURRENT_SOURCE_DIR}/test.zip") endif() if(eckit_HAVE_PROJ) ecbuild_add_test( TARGET eckit_test_geo_projection_proj SOURCES projection_proj.cc LIBS eckit_geo ) endif() ecbuild_add_test( TARGET eckit_test_geo_area_library SOURCES area_library.cc LIBS eckit_geo WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) function(ecbuild_add_test_geo_grid _grid) ecbuild_add_test( TARGET eckit_test_geo_grid_${_grid} SOURCES grid_${_grid}.cc LIBS eckit_geo ENVIRONMENT "ECKIT_GEO_CACHE_PATH=${CMAKE_CURRENT_SOURCE_DIR}/eckit_geo_cache" ) endfunction() if(ECKIT_GEO_CODEC_GRIDS) ecbuild_add_test_geo_grid(fesom) ecbuild_add_test_geo_grid(icon) ecbuild_add_test_geo_grid(orca) endif() eckit-2.0.7/tests/geo/CNTR_RG_60M_2024_4326.shp.zip0000664000175000017500000110401615161702250021143 0ustar alastairalastairPKS{kZP<CNTR_RG_60M_2024_4326.cpg qӵPKS{kZ?8RCNTR_RG_60M_2024_4326.dbf}]yfR٪?Wr[_`| 78D (Ybl%rdQ&AY")ZM}Tm`\ ]W%qzkd+!d~`fdy}~{~;sw'h~t_(Ww6*j)_7J+OZuOWhv}=r07R*SgVXa(6S BAMrw.]aX}ٖP.+R3/ͮnf$- CGHY.[Kc_HiyH^M%[Vx$B 9=!Jj"$%J"'~@j25JXO<{`#>+ԨP^;FBB S:[G(Ԋ]緥 59oQS3//_~`?_QY. f,=ߌ Zȁg;S,ć (QP ;ēGV_A|lϯ\a`E6^kr2&ApȁE/^;2Д;XϯB>+Q3 ycwŋ ܺ}5*헒؈_wDkʖ~}!= 8 u<2ծC (QzCW^/I.e=|$"̮ Jȁ8 1 (pQ̧y zB`wꕆrN+C(!>vsI:<:VcaV^!z[A[~REM؞ǾCypk\F>cR3DAE"ك;bPh#]=!*c Eq+"޻܌"3 f T$ARzDwqCP:di\ ɓ>h[Ò0Uc=m G' ?Ͳd.~W':>roN;woN =WhS3D0.doHM8@`Nys:ȁ]otpϢ].c} (@}/Wԣe 痸O~ϱ'ЇgS (Q0q{Wj*@՝\}95:i:/ˆC.[LCR3D_n|yTᢰyHsɭ>ټ.j_䣘i뇽!b찢glMMdOl􃇗<`zw/J\}+\,5e!U/<yN|2Iä:A؞sϋ%|Y+?|HH̀Enjcy-/Pl!>DwkܴN|B̀MK%n {Lvd/ > 9N!yc/"O뱢R(/~!هj^A*6S SAq5﷏E@ܶ,S3BL̀،DN[zIc(`{fF\|NJ-8O^~4;?\`aգ%9؞sAϲ:~y. ]߿21{wvgĖ".ATHn/垏u|NJ (QMқd`^\ rc{~EFbСϊ (QL& }OG j 6zGzZHjŃ/SS[C'>VD^|.U$cΛga}|^l*q)}~z(֑T]cDM|NJC|n:z\-/rV0Ҿ.TyСϊ (q*t#WsM>!ԣȓz؞_/2GgsůR3DQ+. dԧP0gU!5}lC$>VDަˊ2fA"d{󋭼j^n:—[_]_>˸rא< 9bYOg65JT8=KoGPkY#(5b;_ÇoQMy>tc-j(xGD=aL=[7@,.[/||zB9W|Q2%aXC^u[ᎨI!ɃX,5J؛//!rkȁm CJ (Q=π'q^2Lk],JUj (tOvOU=a:SZf@V2َ Lƺgxԏ_85k:Pdک=['{jMǮ<ζE{ps߈OTf@y$Td9#{\*U{IPv+&7ߓF'$Z2+c{ܽf ȁEn+vfcQB^J\>odPs/Q=3.CU`Wn^˫>2bXqebPzNFVB*߈cr)o=g:5JT vzz!"Wcn4 J,U2AzuV|1$8χN|}"'^yOjo+j O{jNC؞_",P"'WC~Xx-8f yznַۨ?n[Vϯ$'?ݠ}&:Sq M{B潾M {E8[+GmZϾk/5շ`O~:|XC̀8y{ȕg15S;ОﳆV|>@zEj/p4mCW_ys;ȁ ݓ7U@ȡ֐`s5 *i5HLS!j!J o۱R菥aگOb?yZ n#?^_+Or#VϮL;g` i//""֐[s7~l6"Wcm1{rm}֢f@5ã𤜓Fb&ފX 8섦[cp>!١_;.V V`H͂ ޱԣe(A/|!KUN|Ì%hcs6kzMjD_AB/5Pb@no m_ {HM:m0P!|g%5|yS9Сzn=Cj%L-SVY񠫗KMᨋxXVC}V>W\皚 r;ȁߨ1stN 0X xpt%JC#hCEl5| y¾D5~ƹrxop +!~_|~q? ^;]_61c( |=y~Ct;.m vg+ (1ٍWf7z+eWǑ6x7aXAp|T~.XR3 WcnF4"s%p4Kz[!`m:?ujXcd{ه X\ = 5J4oL6\C$:h~ rNl/qiJo=k (0{wvgv}fÍ)Fχsպד}"v{GV!;5t \~aB&5j~ϏV#D4B: B]+Fk۳;~7}#G5g\jDH^5{$bpZl tc>5JU"/eelEuHj4zĮl? $f2~6|>}z."~o_Fc~+K̙T5B`{BFmoC:AW祷%#\{~vp퍃/Q[ zjt(>r~b?Gy~ߝG̈́ eXo=tֲbO^ D\CNy}Q8**~ V=1sZ%rF#>vIֳr%> 2zG&o_WNǑ"̍`cdbtr/G+U h'wM8;5H(ˆ+ܾ,z`\=:HWt 7&C r j̷]\=B!=&F @Z 7Ye%*F(އfz21{9=RE"/@7Ǐ^|;G/~ыz=z{^>5g41YT稬"c{~+nU+Y\gQ>K BDe o!*gBYSD aڮ·N|܂U\ֹzhk yB{NС*XEv "zV'<|~C`wiC]7$l;}(S3DCs:͛zx 2Xyl דtЁ)ꟀT0Bݨk@7d>1oXz!=s?|g5J4D_ā  3p9@zE({z~aF y֧h pӹ}`=V.`$7n!>T Ḿ嶛5[OBÌnk7}zp}j:KFCls5,X~@~g kp- oշ:gq3χ}@}֐ zjt 5 24֑qy^!P!'>!vY n4Z#ꩩ;L4cu)K5ݡǡyul]!V\J&!z+ j >&ŒtcIUi"P=5 "nֱXou>][SBz>luggwgwfg?ڿ뿚ƯWoP[ C!OpwPȋw=C8VdGejN˃[S3zϹTj`_W zj*H:Szzl"5|+:ͨ}h3-֑CJ\{7ݟ5jFDq{u.a (1eS7fjF:\} 9=>Gvo8QXGֳ:L(`}ܗ]t->lo!y>|P3DMȣXAȋw=%FN.d[nב65JGl?CPy7j2K9կ@N=քu=C̈́r;^ >沇Z[@;ooίJ5A;f5MjH53oC>bL=&]l/ǸazrcUP{KRS! WsGFw1;oCw嶍 5JT?cݹF\y=ÇwÇ0=3Eu RL(Pf0C=(H@ncЗ }V]f@G?}?|>z?=jRKCn~種kP=9Tz̚B Py~ao (a_dG\z` 8= Ŵ;CU_dzgw/4XzU?VzNS Bi9=n^t8OCr\ߡ\V|Q2׍!v$ #442Px"2$7.BC"CFzkJ.YTۿÿoJR|hh Yվ"I&cwo7[Uj V=DH`=4YT?| g'ޟ_ǹ[c8gռ'B5 2:?OQ3D:+Wg7,xˈzW$PWO!q/"J|ϵVqV^䖽pKgUYC+9wQR `ݩm.2t1/U(+PEn{!C_!lg lǮ8}Sy6Y y.;xn|tw:5a71xõ=K ItgVf@+/Ϯ#gr{LMvAjEM؞_İ%rc5Jd;kqm>{>l/02|>jڳAޭIFBx}z`jb{I.c>!`^>/}G);}?:w{n}oz&|#Y.Z5Q==|OrJldž@CՐ[s9]aPxrȁuA}|g5%j"P{t L uCabjTp܎F _MFØ+^މGKsj -(.cmWG<=W N?&DE7BX35 @l~/>5ԶPƣyΝGj"4nCF}6]qj*DA۟el_ݰKQ=wѲcԊȁm#u i![÷;ȯٹz/nPsY>^u)K[$P/Q=Ż |jcu>|u8՗u;χB̀u5\=Aϯ a{~i(C8nr>|:r N6KOOڏ -u7J +SLT2_o >7uaXB̀uXx կ >Tĕ;秡? }V_f@:7[ލ+hIzcjSY.>`ը)| Z4 E~},Fߘޞ>S{GDCzYɤk]^E^ȃHoG}r}/ip^u}m,Wml/1ݩȡujq5F+p7_ E8T/z~|z?p5x>5ޠf@ ]#6}kԔ 1W75| 8W]#5l|֯#`~W=횠pa8Y܌`IB, ###XK I>Gޤf@t0-ߖ\ mOp"EqnJWBz9;>k cd5rz~#g "A䃞c/2uvsv}e#v۴8T6jSS ~C|ڲ :?6" }$?Dݧf@ א)/GN/5>"8U0JL :#ZүT;SV=Fu~;ݫޡޡf@l|W Sǐ=|@nc-jh\ח;9S|W퓏&<>lP|+>kfYP\d`M>f@z2Cݥ]SbȁYy }ì~% ^W-Wpbsqz~5>߼M|~%:]z/GW؞_q[{B#j*DhPDz 6SRZZXH#@x%Ա5.+%/!]n!O8E|"}Ο]߿2;+[Wgj>˅d!yS݂\۫Wu~K>he (@:\3v|]4557g_S=;cEb7tüᑏԖ(_3ȁ[i8<P3D6FݹzΘ{ F4)Cn"yy>r_f@ G9@]}e'G?؞G7{CWP7߾7W7mmSlw{{:xf~}35!5S!\=b[IMΟxj*DרPRElMBB*Z1mdqV8+jxgr֒IPѡ AWm#Y:ϏBڋ)|6BN@ܚ©O&X߂|:v> |7]7ߠf@ ]nls`~=P+U=x}0EPj'"=(ˇ#&>WP"t|>BG|:hx$À@F#z܂ҾsVZ>CQ=l!> }tRE^ z!=;>|KyLT2 |v{؞Guڼrc&5J}>e%s=.ߪ P&e8y>tc>c#z`&7|̍BmwC>5J:};]j2K}o]7~,65J[8U4ɻ'aRn>~{"G:sqE' R?L|1^c5J׈kZrr{~c>OS3m&jtp덶W/Zm%i5|@n z>5bgPbvwٝL(ūz 13>:y؋Cj4pc.˜9 (Q}=f[ wy$R?F@G?؞_.W=Lfpc6nK (ю(L鐭 [}Z U|%rR!-ިڈ xgA2T`.Eyݜdv}vknV Pc@CKȁ]7rYC؇n>y6a^DlQw:.\<#jT<ǚ\q&5 "1_Bc>k"ɚĻ0jBKUr;E3 G6kBoVl`O%P V6\%׹ q2@fAhsxy9-{O@o |/&B"lkVP>1g *T{_ȘUčSY.zs^C;".C2HrcMm"VF]7tl&>!`޳%TƇ^}l: կը)\p0Q4 (#F\yή8; v ]1VK|gM}7`xzאV(|@s9=7!ߥfA|`V}iJaV#./" Cev|P?wܽ|5|G|j.ˇ=H{1Zcn-5J4eҨEYH؟ Mzj>E{YB|޻>b]7?zh"oeXHij*W۸_@cOۈQC'>PbهWaAJ7s}f\g c+}#"]$\c.jPzWц:8jhr1ɛ'?zj4WB|l/.fz>r賯BjT*8>B>+z9z~:Yk%ZzB;]fÊc'-PS ސFDH.[8N43\_.g!V!=_h(Y:|}6Ӈ^G\V+V}ax$I?u^ < c{~]w]a=B^Z\oԜ4S y=*kP5!+ad\Esʆ:Iv5Joܿ}KspOr{UrR~Z>E#=AЉ7PbO0t 5ˑ #6ڼ,Q@l_!W]4s87 6r賶;5%SV}2 ߮֨)q,݈|ұ}ȡrWa$3iC |D=5d7*;n]w~Љ (N>6P<X|>oW p _Љ (1˳kP"ۗ{hA?\|qy'`4dm#~ڂWCީDq<瘿%/FL|Èx}&tʞ\}9=+~.@t>9Y,5J{|GHYKl@ܫ?i%TXxJw\> }P=E!ğ' AܓA}6}kߙ~2Mf --z~u/CM Cu CtodPsl;}pm>N|}%U}=fl`o=q}j*:kZ59o #HܳϽ!5e܎kzi.6=~{= Nz,L-*}D^N}!:>Ls`=X˷Xa,q佡1yþ5}+ >zhVM@QS<HKSY.>W+%_ 5JtL<9~X} 9=U$vd_@J_C>@՛5pzg_ޞ~Biy\ 3dsHM؞߶uH(3v:!rc[C|ik"#J{qh#'.r2۝qydeVg{(^F̓nl^W/>[%j𹴯[Rkoڂf:ܹțw xz(m#{>v6ꄑEzy ~aswH;Wek%cW5W~5?WK?oOC9tO :>l^-)~krx!Vz>Zyy#rSQ4wGYKwmjp.'yw籅MuBPKOݓGҐ@G`1N8 v>z )LN}r۸Gnƅ=ƕF=lwPbGw;5%KqkP=[ :G}vL̀x[r ,2}@{/@} 9-'#uP=z<Ře򑪧&Al|l/lϲzСNCi]+"8\܊ cׄ 25gs;}nOȁl( l@|j 8eۆGWq{>,r? ݇0 dT`v`q{j(Xp4/!{G@v瞏OCu5.u՗3U q{>.r? }xem#> >|\z >>|\z%6x*ci,Hq{i3vρ=ޠf#G9rXNoԨ)٘Jqȡ|!o SqF`~6: YKsX9`yO7PjG1*s.r?}ڧ@Ґtx av瞏@9eK/S31$`gV7u.]ax)+K.ڵ5J{sF82C v〳ߊ!@^wve+v8.x\E\=kG?Wt_1+loN hcy>_Fȯ,wq7%Z?z!=XЉ=Èg˻hA?GX)I=l_4чpPKS{kZUCyCNTR_RG_60M_2024_4326.prjswwwV&J:.!J.HBA.J13cs Ccs=#K =#Ss##cS3X O_W EyJ:z:~!@sSӁ`!CsSc#K#SCKKcVPKS{kZ_%Gu{l9# t&~g?yKF U}?q]kے\ZrQP(ݸ*2h`c{šO@4p-p]B,g*h1s~Z};mĻy%?i:K(XXCǯh. g ,4C>|#R@oG_ܦY| DSΔ*XV~f r`^z7l4M#s@-Y9KA6@,mٞ'~.qLea^XK#~.#ո=Kn<j/]g P(SR`9 [9; @ *S4e ?rE{ϩDX8Qo0D&n>t ]9kD5oƧ>R,AgUTiK0.r| ΓskrD x)iRL{%`^༥Z6~#gtOz^BX5 zY6:0}`AWK{Peq.4>"̕ wɍAU3u3zcHcQJ@SϳU󊟪!46u(y{{lP4e8w7pV;6؃#C,p{xґm-,#a=T={͕gہZQ(~>F) x& 4]:ҫ,Ś4Tu.?>ps\ZYwyjf ]@Ⱦޜz R_ox @ŻUY`kkWٰ ˛9y&r{ ::N~V,m $tbo˅WC w -aG@z[0\07L$Ǥk&ܙxh 4F %mm?x*l(;8sa?}L=-,[/ٺ0A[&^VeP)e1&ZKA Atֱf'[zO}O*L{{磎>`[Z8.{`'b ::Dk|:93V -˟{@_zy17` IFEВ:xmZP?kYH5vk?Ft OkӤO!:Ć?˫ƛ@c}8h-Zk~_ZZCgsv{ZN`O-9 vh]o],U/|(*Va'Wf3A4u%P *$ѿ˛ aաy= WuCUcO=0Y$d>E@ýt/Z E1#{clwQ}2o5]g_jq5F0xʊ_XT*Y#ej:8_AGz=}ndMyABCSdZ=!s&ZmQέ_##?˗nZ͏-1Lp.9us 5ΰɦ«' :%Cz )<ɖ{PJ9<)灿>JO{czVojh#aoeyЋ%M "sXxZ7-^ .3/^@ŵrC@[t՗5Krևd.,jwN6Qyd$*֠)ކg g Qmu\'J<^oFґWG}<9(?]o闞@幚 Q*\dΠp(rvw{fr/!qO}#)Q6#sn.6 2+o'L,51p#;Or0uWwf!y088YG%bAB@i|>~R0o2۸_ zH؎z\EdkQcPgo}ü1e@ m@_o*9TI=`x71đTs::θt59'$ |h6BI|hIMoz!&bGn'@>g@A;Lfz51H|Qޅ,Z;o x~9cb`x2GS9Ö>tqi1h]Qm!Aѳro1ɱdtڄK]Jl_zTSe HX=sX#VA8?h_tGУzڣskڍ[l:_砧ߒş~uE8ұ[ѝ#3AMgTLZl'\~n7K,'vS^]u$%nh.8¶w)hODWS~K gP ~6'Q5wxWjp!]񽦀ٹ[,iBn8L|7KṙbLWf+P@2Ϛ>_pr|рIE=!m63 D.hΌ'z||HHg`umHy]M7y\:UZiܬ _y+ ߕ 3Lu9tֹ2ҧ9O !x8tf~ wf=|FE!ܪۚ {\i19AF>)v"`Ù{@\ ׶#DPqv!I.|H=WE*En,*CJǩP.a}JI#iĆ NBeq5g̖0`]H).8GFPO%xeg$+M8Ur(N﫣BNJNFt.hIтpDy]5 ^?~|*I6jk:|Qa!j|Y*Y:ܠ57"֜ 2BRB2KpMOP4g 87W@ޗjg֢gDl#?-atʮ1m Vڀ#}wz`R?-=g~^t$М;y)W 3$n:IVr<ϔmtueѾ\ol OG1-O!WmMWRS}35 tG{Z>Z&א/!D- 'y-O=8~VD=;yyrzT}4ٸm=u 2fA*S*6$́%<;~鋗NMrPrFh3@zGw8#jHY#2rzءyZ$%!$mPbxpI&=F^ ͷy3Av~XLn_Sӿ?|-Rwm%!WPg "w b`\r8Cʣ㷵im  [)s. -ʻ#oXJ`_Oy/o7ܳrq˟r.G _P2;xCPs{(ij ~.Ȟvw |W}/Aj}me6 }nqy/#}e#]:WZsF6oJ[.<>)s^[N㘣~E =@^zT=(gf9C@m$y˛ocv{]5Ֆ;nsa+wG1n7r|z2KE/P>˼Ds~_gIɟmZB"@k{[SQvaw74pfk)s_sm{{t/*UJsϟGg1VЊ7m΀^~u0k]ga?8ERV wx1䆸:g5vIcVIG Sj%x1g@PqvB +rQ} 6Qk9OSGAu)&} NNiE&꠳!N}h3P;\5߫百%GFx?G|t@kt/OIA_N tU*-DlKMR{CJGYj*%TK Tg4OTF&'&(d˗g/Lۡ׼sP`@ W~ˉ &ӖW'%(?'?gƷMZh}'ަcoLzȠuU+ (E&Pz 2<Ƙ:FE馽_Ȁ2dҒE0 {xv2Kh/9Emz@YE=~`- 2amݥ a Q*ƟRv _$&y{ oJr3NgVmVl&Vxʠi)'" .^"N/~NżMN7fsT%݊۲lU@#.\`X1zG ȟ;-U~ C@;Fp;?嫱*_$s~@ʚ+>7RfA@2zu%hBr _*[8 apPpjDJ $!=wFb @±lbs;^zd{)9& u!ԝs!,9eׇn!I!fskʮ*Ȯ>:Bh4ωF#DY" l=_a]"|΅qSrpMYQJpO8A%<ӥV5v"'ER3rRwtŰW GA)/^ `y#4JC~W_ER>{DP }. H-42$W=I̫ѷU Nʐm4SAԟ#';r *A|Lͯb ;Um?pNz2 ƺ_s,p"3܂S>򐝰^ 3Fߢ^C@CƃCpu- ynOyʯ,'1v&]AK`ˇbWnЂPe7j55/H >Oy4e\xNj(us8dd$CW3@ԡ&zs F:oN(o3L9 J5|ɂ7w;AӚy~S`f|T;ޘ~mj/yqRߑ ssFd?8:Fq6ɏQ&X2$qمNް>aI&wm^vo 5yƏN,\W/!G}FgL[eR-- GkOxÚQ ̔ީ]fN' 6H9G'_G`´-e ,X#37ᕑ+, ,vgPc,X) -,>T}\!q|h<å:Bk~ȡ=.f  6 SCjx,Jzτle//ou>3aE_A,$$2CV;dAnp, x2 }[{}2LCn:ZٔR 4N=cz2l?~S7ϰG_2bR}ZF2Yy+l%FfLr"mmxvuqW8xT5o)^Gۡke7d== ,7L_3˽ ZBwLl\oͪWkQ~Ho֯~ KPP:}N}ސ|)V_CmQo7$4ǀODycO H8r6~vJ MvN>oH4,e@eJ]҉b@ȍݨ~t1[*<ssH 3߼ _3W̬I_Lnىpn$3zT=3 9~#$ΰkʝݣ W;̚;;W{Zd,{,M#ઐy&r=l):߼j!aER1g\,;םE>`7ʗOMbe4a&0t|QNgSH>ʄ4#&͠Fgf&$,n5)W爛caLգhdozSB{j-0|r<<~$i*Z%̖@MXq)Dwυ>~tq2V,?bit~5Fì|l_5 0tk*-d=r+Œs? Z;]\Cky|u)oOM/s\I5z/T؞ c9fq'}ɲN6(\X-v}`#~M>%m}`͉k_`]50dU>0ioCkJ~_BeO>!`ZA+/y& HlKD}!iM@q=ސFKk ǕY %+ MuJD'ח$+Em6 %ދom߿}qڶݎZ\,;&j_E'K2]UVXQr?3şNNil,a>By--8'ũ#/p^#}5$sd83ל]M$#I4??TUlg [H"~,+ `O~"hA*<y4lcnXs9:1i6%yJ;EQK#7? {!<[p-rk|k%kVkEnTMKf'L:nk®ƀ(%@swRAr v;ŷSg<♪ љiN*>2 W[ʯL K)߽Y%u;W︲EVBie*Z{7Ti>r7GD72~Ly&%fKP{<" ezV(POj)* _Ffl߿Lm׿hh䦅(zAIGvM@GOz!zӉBG85=B@?tUFlR{>yFa+_hc7ܡߋ"!U^d,W,aš?=ܡǴ`{". b%}o' O 93!w=!xd ;l<:Bߋ9Z-!ЬK7BKqkt>90v,Zg5..{-}+3._tk1P9_ҁGsK}D] 2}_l娳ju˄It?瘽HԌEk>B3 N\F!lR\} p {kgNT_tA' |F-5Tpخ?1U(] B+6[Ţk~tXt^^li< Xm>wi=\[kzhJWkNAe׬E`>?3אh+oõ4tk잲6#}Ω Ғ({PuE*E2b8#ott_?òtխ fA MgtJAP5Uͻg 2ލ6j&B"5E@&3lx_muB=|; p"*C<[v_Z{?a>׳qV LظCJȾji*[ EKO->^]OkG,+%0ßXP湅%o@P=+q<}@1.,rz-VbCCc־&t7&j^9׋ۙp<*fΖR2ǔe'2!}ݕ Ƹܥ( d%2zr&h;ķȅhWlZ xa&O3puFLJK%m{%LXؿ^]4_Uə,?:w 'o?{rx=r>wrT(4~ oY/)"3de ?O_!Jן2ZɄ /GY{y'ޅhVloX/RWo7|wP! 寱/geA[Px{z2T S{WLHԝ{B9=1&+hsܿvj:5w3g]D5/ػ {4^gDŽN ~z($AwY?mdѯqPI A+uac6]L^—2u7h+!g )pסEkj4 }!RHwޣ\7f5Az'{/e0t ˂zߤQ7.,X'>s/vJ,CCϵ`كΗVkBTBAt<(ƋQ3O!uSw恬yUſ}5(w}{͍Հl vSejWζUgԽaݼ"}uXo[P ~ #20+C@Kg7L^k[8'v vL )ސVWx7,62r={B@[ٟa kPlHizE;$?,ʼn! sZHzx\n}d)7Cc}聐drrӓ0tssrC":ms;d2^}`˼ ¥kssT*N3+; zx⍩?v{qslN[a|ClF&5P:륰S K bӪ:^z~zNd63ddHvG -9ͶK})nbޢW<պ [vnFbςq)عd<*n?6[ȌE&`Sc}NtbLIk.%fbwtءofƖQ W`wYw5v5KM|NrV[ag\x?AQg8vʮ|]t?fheklT'g9=FjΫؾs.cs4`C VǦte>bl{OasV; `7FJqm *G:g^l ǗY `+OmV]BCz\p&_0-icwrSD$Gy0vĔނ~cm w2|`g>]ݸ"c}豿,$M4;F)kO6l M4wzlӓlo/YЉ ~0"[ ;W[읩6fs!@qd _$'TzkJ?'YI2Va׽&-8=ğ]!Eylf9{$e&v>;hɛ؊nn.61&,尰"|M&UERdl&߀~x`'Z-ݹjiilR_SjvuplGP&)6Ž_fV&]z9kb|)~d,pƶbwV=vOŽBV목eVcwf#аhJbG| -Grժ.ulMl ^ެ9%F.v]aG.5[ EbSabCC<v^0__壒$v++Xq-'|b|y#f+vHYw|v{@s;Vxul߀ַZ#o_TY&zd촰7 Baahgl굣1C9b-4}bSlyح#۱l9-؝ȷTlZl,̼H՟}j({=)3QmAp;{݆D:VaSmWu<\]bӷzH|ߤڊvSN%hش[ i:[l\RQ-6oy }l sP,<_qi .anںlF=+9O/]M\=k+,P&i8]9K{/ެĦm{A;sŶ1A_sIp'.bH,u]2؃ۊALr56@w%+ܣɕ6*ح2|r% DFYoo+hk)8v9mln3˄RK&rG%6{NvՎx}®I'bJȑǦJF^Yl/fVYl]u?f'A>)z 1YiAv\]46TeK@+~bC{ʫuY96U6狡:6zmN8mpWlj U ͻA]@ 6!A~] i.6[\t3r&$l ~W<a1|ʳ(m]f"ؔ1c'N8kح$>(6'äld6CXv^6@&BdlTA#9m!pXstM;ZiZk=Xg5|_sjb7>JO~g*'޲=D]]vas*6o=k_!0Mtߌ] vCǮYd9'qx3a~n X]Nw46﹛yF whlCųY?&[xv+%d`s}%GdlUz,-^ ]+76u,%g$iP0ve$lnUlյ9Gmk*q}6\d S{| &݃F;6cS$[?u.o}/w9l[y ߥ*aCgFx޵>9oMz"T`LJ$t77;Z;Z?cgAGd-6mKOlY5 1z?gƍcFM=b{V픲g'Om }sEu-'&;L8vbτ0zWӤyweO~b=@#–(J1KzߙYp+Oh/?θnΎrAOQS" |B3ogLP+'&8pӵǛ f nQ:!=cY-U\76Pws2kD'_@vq骸~a1>.'=->c ?7l(_3'ܜ?cމ3߆2p< CdT{p6λи:cZ85ψz8/y>asڈ'W ~O.{oqp{\56;& I5MjaEf\f-{xR58޼]$AC=V;6htk$i+vt`hlƝ(Zf64׹ꊰ5ʚe l.wt ߃b7d z9Gck[n`s%7={d-5ҏMYf]|m+F]yq֖G-Crv!o9j5cZǂحv (ָ=Nxl#ubݽr_K# 6^]@q_K=l!5:E8*%x86Nk}wbFǰ[MOnבł̳p'qMhIQfY6Տa:-ao][:5K{<}>\܈M;Z+)uˈÌW&%)/Ƕxi36G֝CYRRmcs65V )B I$;-$CJR EEi;<^9}ι}ι^ekzlz wniߏ]Uغ"OƱiӟq{.p@elCbc]x{vi̳WRR}]}7ehav`btZ~r!Zc|rqqTa ̾;ش}#{mE4kmH(ߏ \8Wq$v#籗}ֈݛ?#b#=ܪUOzG/te`ђ6湈mc_߄AyYtckbOb]+9<MT}޽KeH`Ge ݲ>E]66J;Wܑ l@(6մjw1x{6ޞ=/Z?cK'Xע䣇*[PC;۱ MC{c6A2l}M':gh w[7#^شNű'n9K`snއ΄[ZS%&<;_O¶I- s3&;cK܎f~+>Mz\ġZ]YzZ`ƅU^%VxGlbL%6w{ n3>lgja.?|s;Y:Pť,|U)ճP~'飩g`ǽ36'S˰sjc؁?*yCɐ$ >l& ۄ=h6I6%{f˧+Uc3Rr  Ugb9wń6qnUl3mgJDn=6S4<6I4ɧ{*_x;n;s؆'2Q}23#`~h_z[26N㿈}Rvt3O̫|6:zdkbOU#O8ɳ .506q/awL3%`Ӟ{.<$'9i~?ػ;#mdP7Լ[Kۣ9¼z;Eyv~xV{׾u;+9Ϝh*nM /ޤ]?nl^}X56w`!Lݎ-;d26+`]\-J}Vp(&Sx|%$6LfbGl%@ 0Ro4q^}dnt MJu&#Jk47kAؤeO$xo?'ԧb܊_y;؞ƀ7;w%Ů@Mr62xn,ECb:;Q|J{g D֗7`47ysΠ[`]@0Bva o蝖<ʍ|M*8o0oa 8ډtu5ѥRbg~;;}%MGW>SõX ۃ%s]F[l76gd[;ƵK*&'=bAalo/ah IzOy\Rž+HZnm4Y(0 ǹ!{p 7XuUW$-47Q.Q?Slj&Mv j؄|23[Wƞ XI?^{x۵bܝ,pb3uNl3ǎn=I~/vo;XlÅ31-iQv[b/]tH}/KLG/9\?8ulMksN/}sWj Q4lj?@bW 6#J#? %}uL\r%;xfs@fSNt,nur"̅XTz4+ƇG) .R5ky"fŝ \0+ Ghn%y)'W 8L >i<;f#{/9%/> ѡ6iUw_|p:ohW?o%hְtc#_wjoHROpKxCW}Ϙ0K^Pzye8ƍOxA 黎O8peG(Jf 9s]v׃rȺZ^pd%x?ǞT-\4 ֝C.ijr λ]G5:ɾ 0*CGhλ{;EPӟoMrT+/+ ~rEckݡ{k]`^MY7Sc#d:Znɡm '2D5u v×} 9Cgyk}yhz>.Ԯ;W_ eGHծI.t\[-֠w6A4E!8B%Cgh. zU؏e~&;BC#Z˼TE鍘|_iXy̓EaB/`-plh~t B{=}tn_s&[8ĘC[3 QqjG8H0gtu0rIs=7vjRu#wvq6Ș\y_ wŎ AHTՙ n"+Ȓ?U|!;a=4!䀞qw,+H_uCC4O9 w`ϲW7h& z}Fmi0Jm;hv`γLIE~2tpl _@}W!-/bI(9C/i%bq OAmON 梀׊ݷJTg69* uq( ,k%q6.y: y kā3W (]m놣 -4h8Bl `]r՞ -OxFT_Z(}FՊ{yraw7yTnn7S͵=⾁ 1W"h*n .O4W띴OѠ4#-5QL/Ϟ/߀^* 7ݠYD邩˧rߣ7`ԅ5ppLyHyoRHW\`Ɨ;U>XB^0` 6oeeZ9m /{λExkJzcπ>{E 8 (GnhՃsdE{}zviq-$) ӰP=l+2Fl 9LUKY=d~c XêgS{;o.YwֺO,-s`UsP"hY I X=LYsJ% n1v% Zn&ڬ=*mC;HVG|kݬoy;Zzx"L-Py]*Gyw8IL幞=$|ȕ?QS'.xg,{HNMZ\uCk^֯KxxR;\|>;9"gV{CQкm5m N+PSgT!Y71]= Tz*Er~2xh c472VFau1sbC|N\ako_nmG*2f06X;6m꟡ݬ+ :aeuyKz}w*7Ľ3"ch0B I݇CeX@NžѨuܗqkVF:@*ԉ?uGq $`r+Rׯ{6&uM &ۤtOhAz1+7Y.߃Uo3փcWQo厣'DݯkHQ/k8a[]>wcW%@j]x3:#I;Q?zoǪh?)? Hu\aVq;!1+xNvd ?[ 3{-a. gL/Q{1'Bs{!1f v Ђ\76J1*o2]`ZjՅ0U0cpk)(@UJu+j$gsVGsjüF{vCxcy6 ^8CmHOq|{m3A/OAݍ9$8"g ?@/z#?l!qR](ZdHou3D-6 ~_9uOz2FzphI䪷eA&T'>zꇧTQK{LGXrBi1u҅@,#w̢3ɍ]{ l lY,J $+lb;ОK[a ғ=}j(~v~TXC `є}5ZG{PP7q XEjx5y9:\ЇTaWkWhk_,o%2`Qv 0o=Qqg~&ЇɳlWMl# `m).kHhS֖2cOl }HGǴ/Q5_.4~(իl!\E 7$ɜt `PVgJ/i`᝖8*maiM/=r&,\}[@r"ǹ&|S͟glTʹy[&x\15G "S4+uSKH=n%SHu@p{M!mb,>_m5=gǦHOwbd@ګG< ,oL r׻xstzX0vt.C{29#[JYBwzO$RAHFYuf㡕p<:8,lc[IS"p~?z; v=fp_1ˎ;#25T7hM$A^瞱Mf̾iX;cO 7U T=:gg עA<3P@6@p4h4 2DT,`*ho7@kgGu+;}-zUA4Xtf> X#,Z2-`:o=*eȰ)75.h{wk?F{䳕]ck7zz߱xdiCǷ'ϵZEr-Y1|KhY + %_8jFF)ĢO\/Oyi *na:;'?+{Xc a%k= {X&NcJa~^;k}}xS\v"DÖ+;d*}tvv얔9:OM?orH49ۯ?.kH/ߙOsn.zX=X`c9 3*Eu۩ht}Q_'$g_[ ˤ$ncTY}Ô޵w 2psf7G:34q)). ΔS{T rE^^Ԧ2 I8"& $eP^71krCƿxBf,:i s>ޥ恜hzUA%`|QˆibTq1{T3* Y+(?C* jrD21# GW o +2<:w[X7Ih%0ָa꿟CvJCC&wsTUm4֗)7;(qTd >mo/% 'v*OMDìX F rE9J}5olGwX?ɺ`NNQuD9@`4]݇v,2N̉E H56wbYź}U@))~[Kt5 mqU_ 0\~]R`/IS&_F{C/YdqK_q?_(vU-'Qf(1mbA90 1TI)pI*/gy:Ծ=M(4}ܸ/]b=뾱/eT6 ueu8wEa-~3MxD_i@w\vM,CUЂj@pA^&OtoUkâ=OɏQOm^ӆJqˏȢe;i ʺfVrIW2z0ݟwVO\CUzΤQ+ KUx,L6}Zй#]\򮠁^uX>0pNuET0j}-S@!"K'ci9Fz;)՞h>tWݕ] cÂs9OM&x_DM @4IQ )f)ņ0rUzW/}MݦxdQ vKY^= li G3Y.@!϶%->Ҡ@C~~7u߹WJX^6-R൒wԋ!ßMa(_MEdB/39Q~47vMS$c益(ұ#t$ g}w[L XQh'1 6zsdʤ:HYFt8l`) ?P}mIzdPh Xk,N9L{J娺>[t]ymüC5Kv0]o.Л$Vv jT VN>^&lۂ耪lx Zk4H(`$xDԏx;PWA>&>kwY`u7*S@y%Ggm J!3Df7hc<,J_I 뻫Su s)[noFj/m>dI>)dު3݁7m 3ZNWl9|m] ۞~رZ"P֞pR&hS05c]<q)ZF~;;pB2y箢otk0fR鍀!&|0\ 5aTu`]4%E|<0\}h䕏Z|iSbG4`v#c>j3duX{*=|t[7 F*^~ -34Tcd(X;Ιw*d @H .R?5Y>yd4ʁ u&f|е{d+ ytpzPq4)Ruu}ī*̼J8R/X҂|Kl{u.^* 1f%Hl-R Ou0JӣK|Ac}+@aRcmz*Sm9?E*e|~T|׬ EIFt@аI4Sfo>SA_W\n5:M8)ϭ=t\rQ㖿THQ]^vfA_OQjR?JiZU*[^jCcx? }) B]V_z'η""T^AQ"z-ηJ@yp 9Fee:}_$3 P~4TaRh=Jڻ*ucc?@/n=D J7Ze8 ͷ&i!h?$rEh`ԋicWM`/t$\jE}?l n!TK ㊩"dηggϣN5{f_O[(K&+eJ3O4}~š_>R,,{Y(T{ȍ8>9H[3|g WUߌK+9ҽuJe!c믭O?}LTZ[qe:_2|0u*dOԊm>hzK^yQi{+5>PFX_e!U5m uyNNoFGN=w7yxUBn\?"*7yFviS!׹#pWyYYŔhCojh{ES45'yF0}ſ,Q "c7um4r$>~>Z_U]hAbuy|B?`:g15_DIO=H8 y^?\C7.RG o-po_m֮)x;ſyBoG!y ^G2;}‰HT掻뮗؃|%6 ׇn!B plݮn3wM~cy1] X d 16$O%MI`m%Y30ncM d]6* $e~]c`Rm( t;7Z/SؾT@ݾqu"n<6Πi`8->:zCK }0zHcs%K||1~UOEWXVf | ~6 'JX[)92-h jGs|Dy_nkzgJd֭}s,h{϶^naָ^ 6ez_=o[@`qypO2݆WԺ]K RO_.\7U}/#yQ%7Q>_.otEW+6-p̢_}>ږgG@s~%HPk؟'#y._RD%rT^&\aU(0)mQ|{Q/mݢ1=g>9bF^؃ҋI(@tXbşcDs$VшV`:>tHoߐc񿹩$5ZSf1{-}a7g*~fB^LGFZ P3s_[ي}`ü|afg`:ѥb[gtQ|T>b,JAv)݊["wLjP{׏:e[i)TAiZjW9]1ׄ7W ȬZQKT?W.Տ:lߏgLlADdN:1܅t:i#jO̗4> GT#J' ?eu0gP׀^ںY;$t3WdnS@fK\cm>o;#AԐ^4yn<~!Կov7 c(d}GQ{SX ؗ?MMREk<':5HWęrhNKg/Bn a2c<4@xؕT cuU?󋡻v>ؒ6ھ1ao<̕okAJls=zR :Ϋ6]'do1m;Ԁ _x3 xX!@ ߉d=D2%?&y7.̵Mjio]Vt?h|^u!OTwvwUQ 1w]#[Ȓoxg&x9mAbK%zDvg-$~q15%)+:SFdCG@G=[3CӴ+`>h\lG=zE\ѹP|6SAm6 ?Fv#O#bI}7]ʍl8+K ⣬W{ʫ<y~ Lkf,(G(_:`2]%.ODb+0N״8d]V)]tLu:<)gCCǭ{aB)%A'|r6Z X(۟TZM[} .--@W Q'Uv!X^Exfȯ XY&Qq820S+u{gujQyVf/koq~x7͌zo6م_N|;cnڠJ TyD6XbxH׽ =_j)kgɘa<\d})j_Cy2@O"TS@WjrW|. vh< > e@B1{Y-/I)sF[k| r$ϛ!׶ߴXg熹68B3Cf" rB_@U MصEYT 4uʺpu]`MnnTҲY{G8Ce~柴C(}tn^^Iy%G Y+g>puݟ<&,w>-"I]"[}>FQl <^ ,S\A%nJ5.;q?GV?17A}e/^U]u·q>\==l]t=mC=S[b(+@m~YVVq$d_u;V^Us4'YJ$mpb% I9 Yq)" %:*?װn(r d׉nKsȩn[Dxb$"IYcϭ> ݤT>]$oO;/8Ϡe/ك+aԂeh|s_fٕHy>|lV̽*Yʿ)_WT0"? hf!1N8,8I6.?DZ?kу+ 0̌0y˕Uζ]85Bt>1q" y\j~.B~g*wnʘ墡;}NҮX_+F'> C~LxG7c.ݨu<`, !W5&@I4u*!4~X?Wl4|11ޜ 6;cзJx/| Jr%(x{̐Yl$I>}M,W'adRSZ pE]T NxLa&e:K  @ϩ0Q .)ˏDB)w53! 'Dg1ϺKE`ޛP|_08r\YK2aGVP)o‰S!˘DZTrGX $Aw3!qE;r@LqAFu.GniӺ)"~UrG"7G#oR<<])Ӿ 2DOA՛о0-FWhȑBo/sE4E|nz5“G|Y;WsCm\ҟS-e_)S;h${gj{t#sta ʀ1:FTlZ ڗ^q#Oq=dª7O缝G_a5ZD]5s;F[Ы9$_^mu L>feQcZmݯ= G&}kC ]I3[G(h}dh?lJQԇEolH?e4Z~_# ]c 7˚t+F?t~I:j.s`6) X+`ns3Kd 3L կk6Ss@V7DKS5IeE}E{nCЧ!?&ϛ7-k 3 ZE}Q>sSڗi'lhдPI)LV$ŠHAHRQx|8<sϙ^<̨_ vB'ւs,Qk#Omq@i2; B®aT8;̓ y< 3XtB?\bP 9xNYaO,p+ZNf﫰D% %`&Wq>{0꒤=ƛp?fSIKIj :X mmb=2N Df~qnc[zzP_\QH6QIf٧X j җp m߹{kJhmKO?7; BQ[ǯ |S=b9;9 ުs+_YmbAf /OaGա 9W.Dc ZkLf!wj)n1C™&y> F3T@I(WP`Jc[C{2)GtmOqx9b~L^xwd϶`o{ǻ23MX;qicx=۞h7&TGDWa60>ӛK⎈E)q $sdWo`?Lp( ΍v=&.O qb܈HDSQo2ʐ(᫏ b?жD"'(\'ޝ ^?w ۆ`sb}%:A$E)KT-h>mğ q'|78wZ6ٸJgPOHfJ ,Q @p ":yopMhZ"Ä-ar Dӄ-!dv\]=c2cy~b6׮Y7݁kڷ"Qivg wq\{>ϒ5זXfevOk{R^x D(o|jk4vK6`$nTxۄ53 JMAF sRT[z &$Poo(;1G\S)דD57DO߲V"@_7ualrw550jL,xOvP$yET~rZ_Mihc.U'ngWzEp֓0^7@l5:O4)w\d=3 7m ׈&};;Gx`*_XX u (0u B[y yma360O~N<^lzTDH~z\Į׮ !Oj̅8Z:%ToA2]S!KI\اUTrF6שjFk'[-* 7նFblq1? tST9luwn;] 铯q%W8aP.ĹcWLAXѬ}+#:;ie%WCK :KN{3a{n MHKLsGZ5݅J"sE/]6ɟ ڼ߇̵gwCTB*Õsaa*&e/${_v̽!*)>io獁S_WU}޻ qt{ߩxmvxCZ]q9~TE,DȀy+wr {3^?*IS3y}kC S\_?uɩ_nljq=xH QӂTD_K BDԱ޶ V-'7k_'4E=Qawg͑;}f"߫-#n!6\-hJ/zN--A nHq쪂IK誙 %7m66Pb&ho`Nl4*8,Ӵy\9vY:pv ~ C=CAAɡ5 Bmz1xpﻍLMt]Klul ~Mw 5pɹTNDXckM,<,L/ [s6ya’in7 K|3>=15M3qҜbI]ܔt6)XL nZ̛]u|i0s"-lT~u΍|3GKRݬiN|iO=%ȌWy F42&&z`E V: os] UɘR=wfN~" 1OŊ5}K3K>xGS/mD~̃eƘ#{rǼ&ѓL09i=zfi7``?lL:{HCƘ1u/P0++><1u,W/cL.8'z~\fx$'=3\|f\^v gA *mLLLL|/{{ۤT~&1kC?ΠcnCV)V,5@1yL>8]yR Rf+OE0k:N{2RZW_zb1#ZYqx|,.b×/qъn.!c{G1kW'^ VjbMt~X? |0g\9~d̺q6}T Ԭc,CqR:_ =V!za?TnBbTM)[k4Ls)Vé|qqa˫qyli-T97D+s ω?k2dYhCy_ w{aT&0`ӵrc!_}z88vn7DMul/Ltv7Dѝ1-m'Lѡɍj 9No.{O9$(X}#0%ez <|~׫7c/v;gnBH)7,AI7N1->"O%IT8SK3eqT%孳.]:QƵNZ( k9!8Yi!Γ0Jṯ<bQ#P'ҵoIIJ$-5 1|e_ŽUĖN/fneͨ,|䔂wyxr쫗2l[E+5\7Ej;hƼ gN8yJLݵE33Yl0!XrNnW*N[G=y\*0ymu{*ak#7xW!3o_-<+P$"*o~4˒KW@Ǎ?2Bu&iFeoo.A'sHp)ːTlbsQwLm69sm\]5iE5jiŨ&$2|Rӊ}~~] 3ג0FB/)waC)(#w(EmɈo3A)i$ļ0M+f".3w7W;z\~9 񕜽p Dgs!Hn%ۅ.}/6ɣ5RbЗIKl$Q e^%~~?웄ROgL#h_owt#TŠE`Dw:?p^KUlo܏M/xW<`vN˱[|>;b]O81Te03:z{Bu'x ?m{Umbޯ;ez<]*9}u M1CuZoH~W1qFrNOZ 1#:`UQSqSӈR>2aLa>j mf}/GՁkw)>q`4|X/]suG$}`bgنmK8Ń%voMS|8aav_?nLVIKn,f!ဆI7>K_O4ZY;[oEF,c֥c O;E|Ƹ$w)ژһw,=tUcvڙ{u13e`I 0 i$crd*-=dO/7i֞jH@#L3?l1Qnn{Tݜa9Ymys_LԜB ٕƟܩd g}L~#3I.3/usoU0ǃxADc,q+bG?ṟֺ~N;bf1zQxW C˿3V n.dkIF,ܝ121^L-m \K2 dv޻:dk}Iqk{`Wg:㴃g.\1 -1uiI|[rPKw &"*MG]0pL_k`lX[ګl/q-n傹=bҒiK]0M5 fb-&jN4AQz*8i&>|wW̻A+aE=C:9\ΧLoZ>ׁ䊇,gb|bӖΩ tô9YONR|dw#%/l1-NL*v޿lI`Տzhd؜%Md(,-~a_ --‰*-PjlYi;kk먇5{[\8% sԞh/$7!x g2=a{Ib3\yԷt;f K7jP迉73,t&lWfA=u8=+fW21Ĺ&Hs9lb61M'sȉ9tb:1Ń's9}b ?1ƟL&swۋ F\:Zs,ljiWŽrʁ4UŤɭ! Ɯ#͐'X 30a'ٜN%Y*M' cSʏf_k#}Q3]a`p9+ Tq̤,FW8+;<"FHELKD 'ޢ3^7icnVP~My3{ 9~Ԑ{O쎧KcCOڭ& q0"f-S@zȄ+=|4ԙ>l+9'Ir?W9&{/-n-Vj1j_dV/`9+]'(>*R'AQWT|C8`{]_QCoWb6;duxG&7|cS7-j/Ԡ2ۢo3Y壬T"f4I jkBʣip1RQ=?_ݜjdЏ)6gQFz]_?eCFzqEۻ{JQs\u$^=Ձ2K< g$ҳ&'Dž-Q|۫]fK 8T 㩈?*{-i]p.'>YdWKtug xj!ngKMA;"Ӵ[g/lR?zE(&͐;u#fh>?R;i3m$ty32侌 h+ekd6gj͑p n3G-6)e'f׭Yנ~םX!Ssd]|#Ȭ#Eonsb9x'g6+_߾"{o yWDAYs[>|X}2GAv'ujIAό;z}bW3_Ez\7俙o}0L܂tvN7'ޘYq L5#OW}.%g#ߡ S.7G,dnTP8Wj0M-5Mޘz:J4޿@։<oWul@ʯ}zT!>XdW}q7O . ޘle 7Uv̏kHAf_l\a5O9FKBAzpug7U".&蔖H>X_|EQF}0:?o:,w}Ĭxhvz|KƤov[_f?צh{cv: 쐽 ԯ˛ߓgW}=s(/ymG]R vy; m\iy ״jk`<&Q1Wh[8EI?],`֕I~3U}7 |e[JU s0S|9#KN{^ukpFzplfecJqVK̊~F vDz?^Oy}jB?Ozb`,D=j$ 7X5Z9ӆ|^Zϔ uW-QM&?6 ʎ|g@)VGE6RDW,KD-3$uqY-"#O~[)~![FƦ*8?7NȈ#hӫ?{XsΓ{j8Ap.q./[fa3.^eHd?7vy2>3|uo%lbW*~ o>MUȪ%g| 8JAw9`tIRՙ$k:a6eSPeBQl6mq8M4u@:%ْȎ?-QլJ`f96k%yNJ fGYU?5#vnr.o( W,3?8Ŗ(a;;ȩɵ)Zi/,? <sfhYs{[Y}Dd+I=\Ԙ(&f6KpVd|=x|dWw²\`%907m: q#PںӰ҅ ݀ .yq\XI82f-0:mܪ AyϷ_\o+ZJoöٖ(p9,̮Z-?'b;pƧ~O҇.z9y 5{RY/ݺ΍W?CU̢Ff!<41W=-+l])]M63u?\ͥOa辴{S烽?c]eLSB~ h"G+l1_gz[!J3UyV藷#w'%TiJp7"XS- #fqy.n}Au#J%nߚX@^(6^O]~m)O@<}W5fq#K 0msu!~a0X=@|\i+wݚpNcLPf}k5M5I )miZdݭs5;L *U\j8N/jY/Y!֮qSLOFcM1Nwn:;xJ1#3$FC0YcGA9 :<H`n8ONL?Aח8NpP0׼ RzJ53k< w~pR_ߊ7pW먶\(t򍳐ԋ/Z`BBF`ɑ?o}HsTsv%>{6p:f`RTk[_fYW ~T%\͏g" (҇*-eL!W^y[a*pZ+ bwZLy ϟc[i%b sL.{Hq__J ]G=}U9SGM0B\50{MѴܕIgN[cӐn)<A]l"ف]?3yN;;g|0_S9bZM_a/v?~e_Q"0I}k=G+'aN=ڞyjRBG.oɋ \c,vEt}-w|o-A Gk]$Ĕ: Rg >'7ɪj d!f>A[*rC<~v"C5 t`pm>mtEڳ=Wċˤ"\ꊰ_Em*<# ؛F~Ed}l+⦗pEՙ:\.sקglQW\W D),>-um!SL|` ]:tkV_zW!d\ ~pSK$ұ1 "xH= .MxDo(nBxDA!>22nwS 7.CB֧n"Ѣ`c>w 'KqnWT|SםEn]3>$ÿ쵵$%m-hb.܈p5Mq5=;-T/6.r akH]_o>_flOQiSWОŖF[#p׵~c,rC{i~D"kEŠ}ZCD6S(J Qўnw(ء?Z>[|If?L15|LFuivXI-\샯M6V4#Wkb=,UC({b&f~*=@[zѯib9&w W])3I=A׼NBlok`ƙFIEf 㙿$ͷQ}b\.$=BU)Ȯ(ICt7E$HԽQhE\ڭ SuniH!>嚁G5bwy]6ybB}9zRɊ{6Ӻ.F5}T|څl!UUnt@qHs)u?=##2]%>_ ! ZA}7g(4fji|i͙f,7"j "-= [Gح MBчe R;Be~P F7\OE%+˅WRόux|Tthv;'_^8E3*1qZ7V*{J<~#4:{L%YMN Lïen6#oLp+ tޫĜ[mYG>5+O|/k{[X.%,,f˞7C |GESw+bk-"l8} CآvI9;qe^q=u I@|"K} %̎hs9sxPUt`C*}gyz2C:,I)97jbQlg<%̎=}!5kQt]qNcľe:Aڧ8`pe΃ON~ Now7FR>`%ddWHs-y٠iCLنm䔠I/xsDAsKT"#wĖYy/7S͈!ࠃ6QPgvsyFKLΚ/^]`U9J/ZZh#Ʊd0n%;Dw 84fm*EH\3T6ׯ |h,u$i)B2*zyifkfp N<7X# vEj7'Ƈ#,I'!(9! W%jW/ \fHC}8a̒7~8`͜tbvoۣ5^kyACeg3(CuϏ iK}`|9P>BHO?Vv*U ["';Tߪ3Je7/2yxs:J3s-)*K:DR;~.] B @^r"_9C֛QAv "q2ke|NX6}j"f:9⊓Ysnڢ{M [ν#B7T3HDa5x}&JslNySYW"zk*7pÕ?߿qάצL1vNiٮ &p<._81?Y B.VJ|$KҚu.Ad ('l~E_Qyt #3+wLK&kb'q8}L:qZu׺.Y_:^o YI 싍mw]7lhUKTZ!5n ݊Fܝ_#X֥e3﨏|||g] ]w>"4 f?kM>U_{xd.F{oF+HpJeDnW 焁ʚc|ӗq"R*?۬TMaupfPk%!Pqc^+PҒ ƈW}:?G`7pBt5ԙ7)ݽ 폮Ӌ1Gɥ) 'uìHqBw z`eAP-{nl.|^)̙ϯ;sq@;N9%kf+D{Ĭ Sv@|ɵN(M!6b,'djCy wLk3U>?;&Jq+wALnX(`i޺&Ԯ\P^7&m[Qf㒓߮1pA!g{c8q ⮺o1bD~O ^kNarWbEt0mo?tR}:ZfJ?m]ʎ*f+_q#½~%1st)~dQKжIϤ[}q¦4Tr}+?QigLLG❫jbnjRѓh:*Ґ^a針y9閦:xK ߦÙ˸és>ݗɆozjtEgwW{F Lo@Zykz_ Q0Ğ], e|p>/oD=MBahw5˲Y~?Z:ⶢ3O+p/p3#;i~xS(p NZs:g/Cz4R]7HO.,r7y;̧6F| :M&GO]9}/'3+ زV-kKߝ୼3\ ͔88IN/%UxC+Uӑ'}hc~ݳI>3ЦgL+ 3nB :pR$Ozh~xm[mj# _~&d'GNfOdw_z..N=jD_y ; 쳘^σˁsۿxhh(U#/|mU#.t}cɂό 4\=R(p.f R b7C#0>]OCpi(zyeg3 gi7I&twߞeA="6Ps44DzgEϼ]wҐ3Hث GC+_ݟuـ^,[uq&{ y U7[0yق/䠵3ʧ]&p{ J~U^yE>ήz䨏 Xԝ芛!w ys:ahOŴ4d]wXh8FC NYay.52pϫzp;#+7&)z&iJ2;mk:j s7/8A:eE,_A%?W}o80r<Yt|aXs7 `TҞop?"/Z*pz61옺' σ7 ^3QrBqwd }p.}]qp>'^d#W{C|pbu Y+$C.o}tC w07>#L~s?LvܘpAx#P>54${Lu: ?i!Ţk|p@Y5B_9~jBshb,p/s=J 1Mp)'YϘ<[t~L_2Ϙ,{WigJ=i/[8~zUBFJoZ+_}¯ h|  y~ v6s86~Xc\7$N۬* xZX?GIsN)pûF.H6p/,z藶7\_u nyC^굽q6,;{tCAQD`7,۲yg:Mw{>8:w? xYvatAou= ƾB}ʛ7}nhex&GGz(K6{#l&Ga$`vM|ǯBJvEx :,jEk+̛o"}}L&ߋ醄*+=k.odE$؋XF@{Z~ (`pк0K90~t%`ZW5=v: .fE]SeܾV2Gd]` TRܽĀk -p ߜv126 22\dԬ[MBG~>̗ą Pi^"&{g)*,YV,{v׽DNsj8Xl40ÏenП%-LFv,˛MD+4Q"/d4Lq- u~%'Kx@d{X|ɨPPPDI,i35mfjTB!LIiC%($I I4{k:Bwq?/Lɞ U4O`~G NX̕\ "aƿo ,8yX1=i<Q2 J]:€p9}EGo :}tdm% X8 qoKyCNw,}$ј9`ssPZ%1f݊]cI|oZ:`sWKLz_Ұq"k4A[Ż`q%![bm`_aO~ j;"0,B2̥l,qpkOӏ``vڦ}YU+UM'q9kC|WX 2棴եl3T=8kƎE `1-4ІΐJCo]܌MR'IT$Z6bpE"$/9 #` 0cY[>O'j>d,Y6ignM,:tk:ԅhihg7X` k~_Mh:JnU=:Ŕ)V` GtqPlBM!ZYX75b.uG|XJcjaGWu}d5[߶ӆQ:WvӻR%]p—%~I.s%m7 _3A= 09tIJQ^?%J1az LQܔDEv柝03=0>T)|݆151>\- TGujRoQ:X#,5yܵGhD"eHݝu~4- 8]=ܴ X=]X; :yw oPZ}d)@!4zVw:pNSK0;EVLY_gIC2ƥml@7Sptي#w9=5lhVY&t\{mXx uZ>4xehGww.>R r{Tт3{MC^haZ4cI+-tkhc, i P[8ʸr<9Q f}A+ܡRhm1}S:{8 'In(]j-p#hEn08|,:a@M7->^|8+\zoԳhgfhFE^t~n_qxaZ Kt_p>"yw>L?0*ʜ.bXpƪe՛, s]|DL ?h*^SeVkUAtp}+?vZƬ."gWf>d)K>k}9,m^WKܴک=]8ܶ Ω>WV81`$= bJ܇Fz֣7_˼y@=) \|zi7:&4xZy4i `\;d0Ցh ߷EhB5@:?j"ъ;*"Ѳ\D7lB{G }BZM{Ђ\eq ̈ۉ*7ף-g| _s󏚃_ P[`E:趔3W-E8eYooD#G%RvC̷&S]Eپ9vld:lr0%1:vryW"07>W/B+(ϊ,w]hn#NoZ GMjK|^P Bs츉 D+ZizÙ̬ 1%"'gf|VCd C uc٭C'v.8k |i#!4ݹ-2 -LD /@D-vk?vYS {)~o ujز*t .䡇"7=B86sU//{TO'"ſwz./ ol nK[ -LG9M TwCk h[we]nXs}CNg)2?udZ:4Vˇntݾ^:z:ϭ3i?:Koq$Z`q`Ap0G+.r{"gB3bK{ЅuZ/7hC9ya z+XlJ`#fsO0tZGK DbVPP@לAX2C7yG ϬC,s;%W8.UY זjeMD OE6TY~:β?(2F:ftĨ+N`ÖεX`"3\ND7 :Cʡ~esĺ}9c$5-W\`H6`8 Å"y`k3;= OGT?c)>9*n}<g.J)0z^h6XO[ ϒhC"7?C'ΉfSKUAgFӀ=~ib`қ7<оGƣ;?(%0&oѩKC:pA]I9\.j@+x$MpflM?h8` hb!мmn~rF0]ΙnZʆ\xlatCAɯaɧD6i3`{կ+?{w"=>vҍBkDD7?]\T}K7<׻x߭}̳5=CwΓle4ZOm  #ULF-}2 M _̏VZp< oCO!@OO\䩤9Xۿ>px%&u3۴ѲT8@G](؈.zx>*_lnbNwxq]yhӃwLtЂyþh,EwrF76hۮ? 55. UAqk`QX?xtᑄh#+M!okcdVr~yy.u*a-a!O/c螯,[M_]̫aԏ4GGo* ٣5jǴ8AWqރsgl:z}#_Vɵ?,tb$U]2$[ԃ_}fl}nU AT-?m?Cswa}ۤw9ʷߣtUJ\в>ft3=?14 lOSs>|wqt֡MhsoM?oT~(BgB{#hAu].~ =&ktӟN2giQ10tm 4j+jχTp/"kޅn`.<* <ݼOMSugtu\pC[Jp!;4NZ;{.vPPMo :\4c\Z;Uͱ!!Syoʩx?E,XJc"h7E愊h-?үB0ߗ+o e>XՅk:_^=f_Y&,coRg֯ 3E DZAk6[/[i[>kF{WN8:U2vb(8Y0XgN| -D ;=z:`ٚ (hq4_V.&#OW~w_@3}RE聟k^wu3 Un \,2z j)>|;QQPZyVzq?:#ͦq/D֎-6QLCvOkDˎ{z]vQ|Ҵh{_CwĠԉ[1Nu<? ډX?{]T37]+('ˣ!z~mg0Ց`飖5ztW['%,$/W03LjXN+@pO1l Z_=}j8f5茱+wMEZi(rK\kN`p"˦2ѪM*H$[Wy7}ڏnond|k\cRzZM ,s2Z|]E+@s]ABE塿gfc cfmY~ګ//[ճ$1Ɖhl ?ݴG?ZtES';uV;-ﾭ.<3gZ/n: )%4th}*r@d6:aùv`" sç_'{},qu[fVαXf uxOw^ZҢL`}7wWOVuYX◱ "&heekкg ѝ.g3}oxrp;]hI6:생"wp2?Z]EC->ܾMYPn'o!WJs$K厉YM었LrF{w,"o>ZdVc53lMqOD.S-7`}a5W5S'"MwFs!^z1++MU!pmswxywO+˱p1w\<JgVm4w8~3MVDuG/ΗnH[`Uw*YF`.Vմ&dT0q5*#eVoٛ +yΚ- EikB]M^&o@fcI} [!o?=&+ɥ~n˸W֤t0/_^Κ$3~bV{k5#~'ԯ6$soq_4TؐBǹ~7&Q"%CUslIxpQExU'7@/Ė81n;QtU[>g׮յ%a2qF/lH 8ݴZ\\Y¡m˗;lCP@iN \ešמkCx.sh*N~A I^dlC.)ulHǣOPmHd =ӆt3ھmЁ̷g gpQŀ$f5PuN޻[cJSjm farL1[ڶn^9ӆ6,2m|nFCJ}mhX`]i1#;-6,;%b9PDE6TSFs³^]džɣs˜@wW>5e (~mFOF~+Ϳ-nWnFdo m5=x/b2̈^?ͼ/ޙE_fF_ؔ4k>Ϥk9t'\GV8@fy_6'}k@KxfDvs;eDH|X4{nJ1柪 b*LIH AWc(ę۔ȋ';-۾ƔEKwsz7%C疘[ƛG oRYœF&JC~gx "^2%Gsi_`lx^,NԫrBb))_? L1I!{S'kn:<悍3=?{"t'JxӮ?$WFd\(m{>V'8[i.Fܤar o2lݩ08*w_4 t;nOu#iۖf'N.[*2G{ЁrWf_dA ?B{Cg Lג2nA|;TyQ⹍L"+~ ݺ\oVOO*O"I:'"k`ۏzILNk,1E~_cƻċ3卻D0;NPQ}>oqޔ1UnAm3xQvȕDn>/Z)-[)MCWxƗyS dLCLZ~-ZZm4聵Cԍl %f h߯K1{L/D z^kWJpY[-HVv[k!h{u2تSK= z.3 R{ZJܛ6x= Is,Jސk0ʛ6*KdIKu7:Ѫ+xOd ;m סtBgL=$ryZO|Y05fbx[x2v7-rKynA:= ![XɒY6m.}ih6!4==qqFmׂM.ach߿Kм+5l`飂DCv˦9]6.T>PUl,YyclKva/T죊% a2גduW$ `T$ο*K,IjI0m3᷂TKt@ [vONVHY}} OpV]}4z8蘼Љ=; >- xa8A,:}tFlO449 zfG{iE6J44/,E]KrXP7 s% ;&Vs?eIЂvD[3J|lќmI~o3LgdX3O1m _1t7-hn#թ;xW OZiޔJxHVP[E";){ B:ȣRJeWg JB7I##Dߎ.k_~LS֟7׃6Oq?ĵ*2YDcd$MY ǼK63(  -mY٩$Uڹ,+2shZ膍Yh*a 4FʂEHXx$7~kGtV4w xgİ+w;_.F xo9wAVCzиx5_QКw^/?^NtImW ,6.]L/chlX#A`~g9q~A8q7ֹA>ة -8a q>UB Fu9oD%̝l-E+R?>9`d 4ٷ]#[>Q/B="7-[2x&W! $b;;FEY$3QUGxDs\']K 9e3HХ+]rJ&oL/W 3oRfہCM\ޡ㋖}>PftaL>-aNcm8&CޏĢHgHhm[ og [%^lk^n0>.M1yٰEnPks -P;UdagRb;4陴U +zԒӊ bÏ`rAIA{~48(|!IQns--`+ptf+-Z~ #P;FpYYēyFnA-~R9X%ͭ.м*ъ"Ԗ.Ck8ANN78}lx-M%gO-DMn!2{bׯ=Kvx0S+z巆H z/wY3%GǸ.1Mf $êCG9 ޳Ж [ES7m^hCmhmӒ/dM50dhyƓ=%W(%tLFAIJZ`+}T[ +]o۹[|,75"g6Y-p2K(xnNyf)RwWZ̡\Rm~x14{ΙE@~byI]Aؿ}ʏ;>hdۙ#`gq"g$1ϡoݥD,w42:.RTI2 !"Fچ쾠Lp24|!>E~y̸p6_.ꃝM=Mc -/L{6vN0[ǐ])esC˾yp\{ OtgVtwlcQiԢ+~[=Q״cW|ݽGyÉpx2M6t~E1[v-NY3Mv:# _;˯:kG/RNBǒ틥/feX_‡-Wam,Vx+i K^:[֘|)]kea?(gx"4'~3:n^?oj2t ;.ɁL~%~ӒI'.S7V~UB \Z0oO/Εd t kQ+8##UN0gΆVuB.h卲>xh޻C|VD>.]\tX^[V&ݏ Sn&+X5Ƈ*?1^oMf?CkeX6c5}({5ѻCFIԘ9Q]>X^_;;ُClO} Sj ϺFȍC_@>PW,sCJ=_m!'Ty`b·k47˟mqBog\St14u/ gPgOM/*br:mˈ'e x *lth(roOW&.yIOBXv4[c!Ts랺ṄpHBh; 'f:(̽~Ýʬ00S^6i,zYu;M(:jdgg;θ;;h~Ps֍F ?!v ?lv 0 `QzyXo6[s)nĕ ~_㹔g}ɽdW+{x˕6dm%]ipk_+˷o^l!MxCn̼_C J >)ӓpH8[g;VQ?{;R!:'S+W,λN+c~b.Os=^gɪz!ucqe4{X mJ[L=h0~Q-K@{KYxc{klj}6y`|\NfrHBut&iX+\U4W_}Rƃ;Ε;WMB{獠؇%Vkڏ霧Z'O+ʘ(Q,:aT%z wdDGK|)XteQaaxМU .yT-=A5!-ƒogŇgύG=؆9$fe=؃enm&b]-}'lirԜ~uSZ4im!asdFYЏ8ʗ~ߵJՎoҲh}нЎ,[7I;S,`G|/Ɏu||6 ;•2 sM^۬Zv$mgax>at9&7JK-#B5'AS92ӎ/>{Ga~bɹ>{,-Rx[KBtFvx>6over'a5{#ݹ7?a}oWb=`"dS-`\hds툰W/Wʎox \̻@1<+էWYv$mYJWHs\j҆ ;5\J -_# M',7/ }p+SWgs+ mf=gM\PjoA/93r\hóྫHЁ!PsPߏ!C _ϻP[L9֝z؅m[ -f.bKzw6k =])wbRm8ܗY!]KdfqHǨ[s]ǔ2"#U M=Y%_+iMWs G=_Ku΃u2g:V3s-͙[ppp+[mŝilȷnywzhTɘٺ[t%g3ˎ<Ηn'r-x͂wlR+`e/&™է8Qݫk8$Ta:gSTW>D]'<"lc N[r([=8O u&h%+t&)a;.wF3qڿ4|i 7zos|W!?d7g5ra%mr:y>e sUmGɱNl^8_VDY6:v+(/q؎LS̉m#C݉]G=ꍱ6Gu?U`N4u;"Q3p%]kpO0wDݿvMGc-uohg噓-cԏ [3͉lPI+qgN~ŕy;n |$=W]s D8{)@Z&aNl:oMNswe絛-\wX&D3vuTݷcWޜ a.88έYp>wp)3dِ=i>-6KX!5u).;ԹDa-*V@S/j⒨b Gʭxҳl[Hy]4~S#m3 H 1CߎrӸ0#.Ѝ8s([:xc2'bIf! ?,ZIأwa>*?4N–kO4y$aN{mӼ>1gR*[v MMMjpKR}IV\ ED"I!ItTBJ <3̙3gΜ :^Fa;~An$X\xf7^ m|ΗU߱JMn6tüIt3I tn|F 8? a=^xQӨ9tRw%<DDѕ6faU!,p%+ӁNg:Nt"W:%.w\IXn4S:_R?J2n ^N r% ֽ"xۖ]e_^?_\bG*ASoVϭ2=V14u!%oD4Da@%Ws.P"J~zdL ehqv'_|€`ގu.D&̣ŝN1K\HFу(~cNy_PM>y2N~l][?&JzpoLȘ[_N| Pu}3!obq{>{|FO ƻ}ߢ;,4m&rF-TGv (Ui`JdBWG>w[hc ZfQOvK?6>Z,dscQҜ? ;t]ǂ ?nMzld;ɔ`̉;'=2G qU0Uq\̂B&%?gaW";`gάgf,P.ęhgJ]e^hkzL@9G+ߎ.݈ŗtp#C"f+Hht'`~,|wpZ@׽r{iw1Ze L[aǭ7Eo D.連}̍$Xȍ_ /$vor=Ăʙ׌ Hƌ}>XZ um*Eìbt#K9P_N Z=7mAOJ^[ij gz{ R;,PY=~ xi=mnw6zª:(gW@!|W^#N/h?/wN\ jZ `=sq:Y^cAܹ{,9k"Ka3>y}VNG{`nQ9wZOoyo:dbSG?٫r%,hmS9P_EtKA@&f[Uoj)<2 YpS2Pױ]5hƒX=aGsjwُ:Y0sw) wEy0/ch_ApjcTz?xaO&i;[,{>*uh{X;Ƴtq#m}. ۇŦjۍ)A#XgUf_,VA&~xe{t$]ZwxQT]I  ?"fg^֮Ev=h ݰˎv&^<g|)U&)HpySk) 1G&q6sduF]k^$NfxTT_c`|<&iscll9 'ӴSJS/1sjv7I!hEP6 x@-( hE{e#4uwڴMl􀰋u/.0I_[m2`س1V:ś3jQ:ٗL"USb( ]~b Ɵ1[|ϸ{/T`=+OҳdsF_{%i,M8ĹY79'4i}$8q,RY'p='sw|:HL3+@}]#W|%X6 >;^q$їoxb[8N"i;ceor~~n4zbɔKvWnBSwG脞qm[;ܢ{I˜6m}FN*G. `C^=$X[$>6N!mk7RWxja֚4 zϝvX4omltd:I q>Xb~čZȵw0q=!!q|}Ϝ % %FFNdHfɒlP+q,'ak sgCoQ'*o^'fN#IoEY)k7 !Eg$|Y &Ia7ҧjerBZX/9yǥ{16ԶgCtskcOL6d;Ns8 bZϩzsbw>O +~8g]i 5e' ݑpb(&}XKvL#;WpzI<uʝy1|gaӟk&ڊe ~^+Mq̼'ͶkN߶o0L]w蔠.#z573~b햿7FOyqFD|Q\ZW>ܹE[)C_h-ws8߾F}?P8}ؗT+x+{$C콟Q_ؒ(/%iS{/zF `=UW_~f(|`xs }jEg>ԍwr Y$|Zt%runLə#^Wwb_`SAҞ뚷 >sȌ㳧PYk{3v)ow_yh&ɻrnO}I*,c_f~a/&Gq-[(c\4y$. q`4aijhV^nI]lb+ ;1&Lb1>iJwx<"D'|d 8k mZ* ^-4xtM:F 1Ik5Z^ x|x;jxM6‚w᨟J GEp %a,Rb(yA{7e3XD(󅃾)T}w.Hy-ȫ?~0Z96k'Eo9NiqmWfY@]օ;1D:a}>wW}@4| OE';}7T*W"N7ɄXwZ0 /FΨP6a7.77Q-Vw>ޱ7#|_M{,/H2 M-ˌFi#|i=.}m&`?p`yid?4GX^ZRJTS we/`ҵyd*`~5Vi9{[~bKO> uwx2Vi,spY# 5:CZ3Q o> HY8A:t5o9wHgo'"B\q7dPe/q3ʤB]KܽAoOѬ%IYgz_"}%~8F8yv2jxWZj!*#Y&HAF.o|/kX߻+,g }5A}(Q5<]K̷i7wMroM ;; "O h>:1vGli:ϯ3IԠ7Ԍ M1̡7i^BƚIDo/q!s(?z<%=Wp.'XK x\eLGN|](͓?Nix0O{!=m)DD($"Juxq/x_1BwDN WEڞ| bm;P[gC~>P<*r4١٘Z]LϿ>ER7Ȯ@ǏEed4~>%הJ#4$Uu.JL\[UUqsMj62Q]bH=ygW֤}lCZ'j•o3K;oTtqgdA+QH؟?M+%C42 ?QߊBUa/oNvϏi,B>9m Qu//&c|OѰs߸k|7Mh_b%>"jZy]x-$кO Gfvn̯c  ?F|d -2_tQf2{q֙G .w|om9Xzxor WP͊K$e {[rHi)ߙ0J/J}{~=iiˀR=MA"´A[D]SuB[rU!ciu/1p;(Լ${d_ӖCɔDWĚ]:%׭Y~QheweAt w\.*$&I֏hWY 2=~\ }zݘYA1YKIP S6Gt./XS.su)=X؋+!; %V8_)\Iww eb\=m&5-c<R/[ ~Mp'tF˃ [7?Q6r"~GWpatղlC't*pd .7̷}dD;O:'pw'.?}?Ie9A(u&Ց5EFlHdsnB26qA:9b>9 ~#xOBaښG2Qo~=[%G2;j^pG.Ҳeֽt$ЗXbN⸞0Q\ !'dW~]2d1eJ_Q(] EjE:7]\ 7 }sGUMpDʟ?Y(cNbvlױvClO=[+>c6'6A96IlsaS¦M6sll9` 5ǕL>8cilR8tskM{W~YAylo?P0U"DلB{`eS(`mʹ٭iMKC'EL ba1uMfg b7Ӏ_m`&bo? NsG@cb: r$[Cʒ$hgZ L@týߩqrE0Ja7POOK.18Js,APH+c+op\hXDKI_&$u=\/6"<6-J3%:P>I0yeܦs*s[>KX''#'nᳳE"W!<3bpCBsML^0A 56b׹!f]sdn虥̰y &J kqs;TW\ߵ)fn[|Ou39"fsBXˣ@*:>ݽ>95Duy02۵b/6kxmd53/~~ 27bڛ_{DyP,-&w_%<WA`vJ{+ݧq9@P fK7{7`|Mj,m#/W7D58ֽߣp+λK{g5%n4=|P:wcmӝ+UxcMzBWSj g,%-?hP_Ժf)`#qtZm`o]wcӤVa|J1=tX*gBѯ p *C Te~/MдBo6sZ^/V1D8k)z a`3Z@0ꟽwu6Ԝ6CP3gٽYݪ$̷⫸'ڳQˢoP˞P>?Zuw"pF,VLEi,>ڢ q3 TW5 Cd(-K?kNX~)3PzWN k>̖:.tpL=Λe?c)rѵ7ũۊ-~L9FN[=Z,V}{3Yk"pg:8߮z? -p9 lz;4c kU^S9ֵat}Egazi'=bW~Z~=ߌf/,6,C͐~$\t\v /2!bc=-:t.)ogwړ.y<D.=(眾>c;W˜TU,V*ڜ+l>v?E}k^5|a[WfYE(J}To<(gM~'-?Cm15k-/[7j!hIvmŷOzCrUu 2+\}kw}2Ghprqh̋Y.Ѐ}6f7Jьhto겛 #0ݸXs;7m#]:srxߕ WQM @`*,H ȥjc-OI6A}a^a!ԧ6G9Od{E>jcl\?ͯ)3iTX5sLبP)$|Foo+O0VVOIҗѾP54(;Oܯ>)FhOe+[PPm؛r7蟧_ [ \baqyRp=z3@.C<aahlpJV6.c_\gkK641;Yŷi4ƠSjXE14_"0dp޳$ ueL}^b4 Gq?Qi1D;=qHH(R9ǁjDZ-&yKdqן)5 3̷cLBsv(CIO߅a6y$<3#oiduM[ǎK?~+n]yVo7k~ 3<8~y$ riPqIriA1 ',i3S@WO=b!)y3r2U-˽r2\0_\\<VnR{K)2i 4S?Y(kVOcUіݸ}4-Yܳzws2 -[+z3X?ujth [=3=nmuy+oncK=QQzh=X\^Żq&U3f@sGV<0#mjc`׭N'$ X?>:\2.oQ?oa,sZtPbQ[tA)fckRaү'gTk3q{7Ox-}A~~4Ѹe l{%zfNf{{+_%G'm}."&NN+uP"$y9 I>7O6`Zw!a؍ZH(2̸_f%i@Xv׍we/U=dӹhق}4+W"i ;J\HW}Aߗz#vi$1iV; ֿC 'b'Ė@I]M@_室r>k68WRD=2/Sz,>G&WٵFEwp}'iMkO:@WNn$oǒ]нnR ePH+w _5~#.W+x,M!S|4NUgg"_UL'9A,_9^];Avaʥt*g#gWmžjG2m=>,1#H_viNWs%t"A̾Z:!+;Bp[n:.lI rEh_vwi|Dh~A{N;B~qCtbJ;bF^:CkQ䷸޶8BkÕ?kڧ j?!u^W0v#nyr"32#ʬI'-Me@&5wvfl]!f[|Fr=0 ؆܏Y 찿iXێ*xUאTNc«Z߲۔A+h$9@Eipi:@5*7pk7aDc O3>?0tm?,; RU+oOynh@u2w('h7awh# tO c_`kzVaӕۅG:@SZ6=7RQ~GpǕ K!vPpC\d:W_ѻKJ7E@b#qKojd@k.g Oڛ ^hA^I `TD. ,ji/Na2+QHb>'L}_|zFv 9sl5׮<}͂҉'aυϧf#d5g'|Vx3jB3"c9Sݡda)(8o $;% o,Ϝ+`+ךYiߟldhj> }smZRׅ$hwϑd!wUl\>ϰZXlG~K>}~;i;k׬zɏ?zOr@ rlhĕum^||=g(ٕ?y-ΝWBãw{>Ksd>+E93GP`k3G|,]e'g}179SK!CIxRB>'YgCakO\#}RiaJa<,gSpY`*LNmCAߞ7l\n2>4껝n;p> o0r3cCOe'Ւ,voR&<>Edgv0蠴b d+t]__ځ[&+`纗 d.uoٶ-W`e]YF-\"]QOD!ekt 积\-^ʀFfr)YFSPY3"a&ӡvK5[&y`@Xbr)0.mAaj)Q,۸E"e,ʄ'>Ut>+f-V#gۻa"Ws ]nW~CMg^=bؤtHQ>uD<ه풼$Y-ԟɢktw7RNlRYYqQ hvlB~Ow->lx 풽El|pnnBڄP6~@i^(p"MJA՚NQ~6< NlVc!+:U&Z9xbtu3OʺW %`ϑzG{h6x28 p}؏XߓKWSP_ιCxu)ݛ?|}%ǰ 5 $;M3:VJa.ٺ<%ܧ2B;V&wLb;R&MM|65z*؋hn[<+34VvFPIC&CKnځL|6It2,]A1?,N8eWdnI曵ʓ)DiX5&m3UlAݾ"6@5rPfb5$?[YeׅXØSwQ?m$&|vA}Wf[ޏVEp^sAz ԟ#^(ʴ!{ܮYV6s_ Et/mg')ս0߲P1SK2W~Hk˻,PtVS { @{奿xS1?4ai\}C4%b'^T0Tӗ%]c g/PwÇx2$Cӱ/gFϻ? Ļt}*9N\60/S,7hDd7k=37z)~OKO>6 ax8c.1=X+@a\w=*D8k|'0thÖKׇu9Upe.\cR${xu&'1? p&}TOzǻ$z<[q#&Y`?˛6hyojD~7ҥj=СmB6 ygW hc8A!.5c) [4燌Ԫ (=ke ֙IbYЖhG}2>x,68R6޲lxgz8Ho 빙jȜU>0%}ꛗ6آOQq S !xz^_  !E'RLC蛰td3ͬڎp^d/VFPs7GM8 /Hx(O< h{dCj|k|΄4i}M_-C!P:*c) 4_L]d0%{ޙs 1CQGڲL>jt E!-R!jC! e*:E↑Whڴo+CH5ox75QiKLBWhbLV`7LΦA_>L'Dp'o[j "O $xvm]|!:I7^g%4(jRg} WҀ͙T4<80ovy4a<̈́y4c 9.4s=2|a1UW#k$F 6}ʕx^G5M!J䐐;S(Zdq!7stqc q2W7Ebh*<}bfwa=vl ȥo#iVfYCX;UX/4U={3axEӢ>Ǿ/OaV/םPu`aUYzi{~Uem&>C~*#J-0I"dU-i}a$g*nJ\hےP'ym79vzoxQOFiB@jي f17j e*=G{3caO}[`-oQ+8cX>? TlW>`!G78o<1X| UcUT-[]zBKn%&?Z|qK hʢ n}X=gb]w[EUq=`nX p_g%K˫9mrYol29{5V.c?o/>ި͉Ow؃Gݜg^:@Hu R]-G}0ЍT6'1YհANzxۚRDZDDx#P/Hz!e08kםrKWAWswSqH8)!zr Yk_hvAVܹ8w.d/^|lrl_Ewz/0,Z9^>.9c %ژ_ c[BN}lPq27\>;ߛsL.`@I$5u8ϝóPXz&_ߟqY%aT wW0*K`G.C =:;^Bali[ynS5,ӰZc5#h%ǡTr6)Iҗx<^aKA@Pl9~K(YCmh_ihữڷ4tX, vʧ54e24AZ:@95KFp)g_ԗf<}.p䢓kU1#@c$6E~7VuMr=xxPٽfkMdwb鏵ISFY3cC [8eQ~t4 ]{yl]8K:všh?h:րC&e)k]O)g} Wd Zm>Pʊ>y'Xu^+ػU:ݗ+C[n  uBz5B]uvO;[_FN@vth#F(4xw٫i3f$ii2zgfܢr2Ps37u#P 36.F}ˏF64oW[f| E:xF>Κ\?-hoɀ,yVkZJ*}RԪN-LTw+e?ˆW1 UWZ.+@.˵++PÐQ ;d[*jiX4[AY%V*Xq{YB5\}uǻ:7^ѓ+uu|oQ +7^ր#%䩚@o&8PXN/d[J!R~nn>1} MhS?Y>E 1hF>V//M+bZJ_0s}Tm4>S_z nga|IbWzқZz9WLזs6$/˻w$׍9-rV&dgG82Wq &߾"_oYX|:ݔ4E{4ίqx!ڛ覣H`ɾ6KN4Ff?ynV{۟/k r9ϿSz!\p|8gJw DN {mR &gE,7{ME8Vx2ϝ/בz\exK1˅s,f7e[!wP 5F//fe̅?0T bjǿ_27;NG\ MP`r`Jy!b]tu浢Aj1*oNxٷ}Q{$%ꢦV[ y'lgVÊqg:^/UfpXi#9BFێŤq9n|N'ݓ~Z.XO/ èҎ80'@6AX|VtneƙXǓPD:\9_e4 ;q#]O ,dnD#d?z;D,w Ķ&ṛAFVB],pߺR9ʋX ?t qin?ʛug1mz1~'5vHEIh{Y+)MSf`] f~/wY,?-^G />Oa"uB{bɚOD<ެ年mθ"=LG1dPē{ɽ!2=y澰-w=o}4\П&l_e<%tWVG3s6۬s}s?Y6 7s!8UCr1߹%=gtx#)I|y.e`[XoxvIi} >(Ke_]OͅJHS"#^r`jbZuW#+A $\h41c/kIAo}J6mxtz}OO}U5M`&mPb~U'k|ckWǬ>H.< m;g%Fm_-ဘS#P+kLߢ Jy|=If~юtW{ϒC|Twad <*J4g'8v?g&?U߬Հ#j"onnӀOkb?$XK*GԟZ5zxe%O\%,VO'דsS 􏬖'٦Ty1]iCe:(^#=x;G%L]zU>,rO7J8]Ș>e. ͋|`]X8ǿ9;Y^qٳh½ TJUۊeB]R`Ut hAw/~:+Pl?HH}~.{kl KUj]%~J`aI"O w  (g^S Dr/PQ˙5]Hs7犰3~Ceƣ}]*mISwօސji3>Gҽ=`iCLЇ :5{>0h?궴=@Q="ѣOsp/A~GG%E!>ۓd4㛢> 4g @꽓e#g z0H'YY$ Kޙa%졞8vum zwq?sjʹA&߼a3Ԕփb s=/qNjaיA+"3$wL9MO!2%ḧP뿾 ~|GQf쳈)Y=dZ/VK"Ȋl􏹐TǸvlɱܷE,y՜qAStu!qvEN}؅mXIE\+N%]k85EꁤM0O\r@qZ2=t|Qy\"Yvb<J@s;OАLfl'?)(/p "{״17( d!t;`5YGHa =(SߣojÈJLYo e?fYa&h[/%'4H6k_%IkpfߏX"FkFbԁ9޳QJ{4 k_˭@.p2-||f{tWS+7'jt@;6>5ׁƲ3 A5z#?ٹKL IX]Eè']Az> *^uf9ZbP(C(O_L6{KO锦g <iXFʴ8*8[nP2yDŽ HZH`}'5m\oEڐNĸ|X6+ڤaŷ`\Q>lv%bffUz7*Z0 w/[ #xh\ӔF:o >HŸ2qOoYuWgB-33lsZ>mTjcUlLwYxtlpomFyJq;7^|5DGQ KH#>7pOvnK*4D OO EG;^&H>{JA +#-3{A+EDQ׃,4.=<|z?y3gŜ૴e<ziI='=(uCL~_XR߸zKU,yDGiق'MGzzOF ڮ#"x.z[8I@Ez ڢ= 䦣;'bE!#I7l}P\d[Z;*HwRϓ`qb#)`xۦ'҇kևꜧ3YH'{pԇͻ<'յlm, Z~F*k:V z ,\}AO"?s_.ƇsQ&A"!}!a.+Nzݛ3ڋ{ áyj窰稛';)ЅPtuȷW$Tg2Gr7%yeQfr߹A\pFK$Ѐ}x5#> w2)o:4hP'w~䏨> BX]NԖ3F zpk逳F@WBJxbIqcLC8k B A5x/u;jw40^L憬*2 kӾ2 G{D}JA,јm KI{p}}\OC] {G_cw2O ~۾^>n;A6h@YzGH>MfRaD.gɷ܆un)O:x-H$>#+{"H70lpnJYMvN  sN6$:Q''h5nUǝO T9Ijc}~ dh a飌P;`so8 Ϸ?qB60NXĮ'e;pKǞr9_|v >}7ܿ9# Sg&bRRP'ǘ32z\>@kxu&r4"p̴ p*kzy7$.™Ofڟ2y zK/>9h,o\<쭵 iw-Gb$MOcx1&f,qc1ێ3 nvM_f{DZE ygOR1F}TP>:5_V~'<ީvPʂA~fI0j /4LaóxD-,LރV[-:,m@Nl FhB"e ,€.3۽=ÀUZ" c z24Qq"W^(_to6P(4+jm`?{MCl,+ʶ,j1\nY4p NmZ/uqi(Tm8)Gv>5cBKg-^\)gά D{[\ iEBŗhPPa{Sh> 7k lA{l>LK{/8/{)ml (Z@d:c DF Qzr:l0\r!|_!A)F5'J{Jl m!@Mmc DYvT3 /L:;>N}b ͯjk&50nU01~ZuZk9Pz<^ҷu.gb.t"V_ Ӣ/XL'Ö P< \+Hzʣ*wnfVWFTE2+L<ʼXlR{eBAelQ/@"`$GfvD(t[J0igׄɞ*/8i*nb mug%43lJ fC-d#&KXb_##g=*-.r~A|?hu(0E-pJVD8W , AOTH6Oad~ "`7 $JC{.<v$y,FL^'Y+n/7|g~ !drYam&0K.366uKɿ6ByoIc({/gW_qJf~*vup jм@-$ͽs8 rĕ=;qkFF@ ;z1ƥVд熔p1KBZ.R0K"f]S (63}<{b4TtC/vdN&@^[ۂ{x4h K W, ݆r_}ZCvya43`̦"V8߬j:@Zy(ج%7 {:KՑ@T]e:VGOg_$/E}=&Q^ܨ10[?UcO1<|)ͧ}/7*[@`-5Aڎ)'>:: BuF؇52:ؑ&ݼĦ[$^ T#`o_],FPRKsO^kL6ԟ>*74F3 F[7vDp%|\Rp`H^t!f1#$>x}þV nOcϝb!iViK@@*!,C ζJ_⒎sKj@:<(ėgD<5vk?phG{])/b5'xY&?OW 5=v-M^H761xg3gS(H jJn9[Yn #OY@^F)ε#M o݋uȏef$/ʻ 1665()= X+7A^u2+K^ime̡A1f2?u4Ss) =EwVm¸̣7l`;sy>tq'?H^)s75ڃf`v`JoA='ksq.6b*Y-sO?2c74|R+Q;BuahpD =?%k$Q9bb m0efzi_Q>7,)gw t3g$䐼A<+:ьX] bzE( ~y5imBѬ^Ps|ETk |Ŝ%:`_OxAv]2c7tSS&%Oqq:Z%̗VGl-AߧYPfb%(f[]xV0ӏ,i-IW* P%AO]ıN]E8vȹ_ )M^n=U;s*Y]# XHX҂~4Ue0 õ2nUs[upHzuG>򶅆wC\ϖ rۺ9/N-Ћplͨt ̿!SlaEE?{m!i(5Ao(gyOW4ٕӝKyaQ+tMѕ6wqno4%h,2Qk c+>\o(59BCy{e70$7e<Q׫%O]o[m4N64)\utR̝v 8T8hWk Y,7m(s[V->[+>ɕ4HzN 7zN[вEa}ɹ.~ Tst$#oONfI_`k x_}5I]6,{HhU+[e.@z7^r_멉̝vM]7cүg|اwOf>%_ ë<@2쇮kykղ!{_Ѵ&I 6ݜg@+*"NJ-V*;' B9s,_DA.89@>R!8"o^l ߈3?XvE7ꁅfvɓ-EJA/]I~ gJ ~&dC~bk|s6{p1e*/{2`(0pcz۹nsq~Y(y}߻ #󲶥H@$)-\o Mq8z5^t蟼uAG _^sx'G?0i]\ >iN@Mxs?F 6m`dN GHwzݽ8&|<e+XTϽVp Dk!,Fv&N):ākUCA"?zЉ 7=C.'Pչ *h7M|cbvPya?vUVrQz4 yU٧țؿ]ϴtuz-6 [-ni0."?z6͸ oQ X5 ~mu8q}o]0Ƕs,pɚ8qGz\eLp<>s%&V X&ߚ:+zo8ϝsy<w;c:3ii@"LEQ~zVc9k& #}*Uv_w2I =hDŽj@4n@{ {z*1RFSJ^[ S.*x!ҭ0raX^ϻThrBLLk;٧ɞwMTh_w%ZК3N} _݌M_=UO@[L6q,.5BӒfif> o[Mblj^|?`a$SQG2#y[ldw?&uѻ$g'pb FP F/Ɵ~v To@}=ɛl'.Ox\~-f!tz:𼲷m?Fv!P-XA)l!0TÒȳ/Q D1p<`}tS ~ePB ʍo3E~z޽@p N=zX5nx 0z~9_˰VIH4Fޑpyn${k#g!}~[cYk}#ݺ?76+_B'}P00kaKMNfmI{rx,S^ 1A߁PՏgQ[~eEH/b߰Dᬅ] ]1NjA]|@2>ExxT]m(gA "fW<k mb Po^0xN0ίtwQ]O>%)]?=F Ucq L< ##RuXxD܉% YIuQ@ u6F\rOF=ΖB~-k@18 'ȃt>3T!C:|Q#.( :Zvx w!ђ{WtC7$V;>[zNY]g:.<)yEϚKKRD q~;z% ? =4?3IZ]w ػ #SOLI)M,;Wge6 y~t9p Ӷn/ GF,SClФ%O>ҘefSu#xWmrA?-͜+sKt#a|1=f2I2:1K*mR{ 2Clt~K|t,S` s-+& }z+2a96ݖ>u]7#Kra bbdLK4x'4G>_¹ U|ZŏEvV NP9\tLb{'?^8h9gXpHKY[lŃIg iH+2t_d/?Yf^:Ĥ'2b>ͷN0R uXn} s՞.qo*@LЎ?oZ3bVYg)jz+RGCǖjK׀bunݧ3^g)!~*.KcΡlFo?Y-@ܲϖK/=ؿ,|IEZ߾ ~I~q'3Z|vuq#_lh^Y%xɘ,|Pt`qG6Mt 0y?US3"iޑX̗ʥyPF]t֏sfbEn"0.ˇc7^VK5/bMDLS[|P_*, xMD}f]̌'CB)#}.Y]"-"Yy~z3@(}^G.b=]"kzMgA>|}=zZN qz][rP۶#s. K o6fcP0gdYcy$' ^{7BXؓ]w\2̧Ư?^Xq?8 \UVA#6װKW:[e=;"},?@:h_ݓQwMr!Ǯa`sz7Q%~mWU?9ÇwuUxo۳`0v6d2%[0^]e'tcي lE ShS_wxdZZаaX%/-)%޻/:RZ 9hQs&C;"/yOB{L{XMT_R} o{K^ߢ>4D.ɳyTcyrAJ۾jGq>T6H?~CsO W*r+chG '֤m /8WL:XR͸|~R2H ;~ ˂90"dcsZ*&ｏQ/$}6S^CəU;= 䙏V5|IcIʯxOqЪ1gL]sOvHgr ` :8+4U2ERǥLҰ%fL}*n'8W%:%̐9XCh/qQ kzlKC`["+< ]~zœ &&Bs pk Tf*׷~{ڥeah_!!Y- 7aEqhȒ-AO"_W ËBB7 C p;X>Lz!${ /ZET*mnFkTj#Gy`trE!m3Sk M1ǕH1g;Y{Uq8?Ҳ 6w,M/g8Gu-Z9G4PNAJHni:Pw^џo7dQ}|ƨ58;^1T1_G `o,TnE{g)iL6mZh 6VE/;-.|Y}6KOV/BK7@-qZmM\.@ɸm vKm7Լ~!YnŅHܟ3{"|j{ }ooUr!kD} J:x ߦEg>3K?:(g|j`?9AN3Ϳѯܱ#Bd_:@y?lcD_ϏXQ`sZ@i4:#2ŏW ^ ˴!ǞPDůywQcUO$YZGuVZڝUn_jx]4IL6IkH"tJd_+ LGެDSݞ?ӖjALsYs8USC9MˏZjYK?܂%YuEbTRd9eQF;S.YWN%%/OyX ۜ%r2j$ǭM=闦b&3 ^|̡t̓Wy]TrCɢsYT*׏2*>C{\|\&1MyG$`u#v|d-`_uN1aĮL"-=|PlJ?ų/IIopTޠ^w9ux*aB0kns }0^#׺wk./21!w&0 uJ٫J/%k2Q#&`"tIz@ZrXz?r@Uo0GS1( dG-{Js,+I$qP,W- o.{,";q $jΈ4G x>2E9IξjWDj֍q;UY,ٸ8D͚^[}T7q>$\v 4 opt)< jtF0|9xv) &H/{.nMvqJG`-ލ{e & f?xdc_%i6h@ߞn~Ygi.Է{"!W({h@zGk.4?IL \L{; [̟'@ss8 H@…:3 qm8pKȾHkboH11~ytJσŨT-1RJR}qg3Cyi.dz#-IU̗OO5PԠ:uSKؼO>=fcT!S/ͰT<Ҹ߿ssyUܯSߪdUoh08HҀ= q흫$`>9 ĀhcA֜)KjrϤ;6J '$wvyOF˄\+b{Ϭw^#7P%jY̚Yg 쐇u–;x=E/]ܿi;mPU]O|:/SY@>sXOH|o͝_xTf|1/?zK#@ՖZi:c$2T砐 bw(Ce] jhsJ!2 oN6)Au]1C*1_m`2W,GldY_Z_2ub,DUJ[WW>-s|?[Q:쯢Jh_~DVXG9 .h*I!s½K&T H_ڢ{ s77e&q ]hqΙc_io~7s]|-H:׹WT bwTieؓ8.lxtJPhA[W&!>߰2I8?w{*}k?sRNɹ?lL>;7lN>?6S;&Ֆɵgrm\XW߫~AHO:NՀ;V(m5t+Wʚwwr@%|9B kgSo@Wm lHdAcT.m& 8jW 6g hpB XnpL:h[Y7 {T|X-bgKx;m_X`0WOc['_*#, 0zp=@* I^a.wRE:fx=Ǔ|ϳY;n5ze2[NԞE}7^l#++#9 HWjxW*Kox_?3#HqB۫c \\uHTW73Q`Aj~p__ >3-D"v?B}~r߰'|<>|GV{gi hkۺQI-->9HA:498L'KBS2q֮.mD{Vy<- c0~u}OeY@D탔Ě2 p[1'}ZA2@۵I6)-# hWW{+@{gyPRsluC} cU?$ <>1_}dMOw_Dx2UhV7y2d;=ی'SJdV{f}7vlE @$h<*j3+Q|(! |WFZK ^ZRPv<ꏟMx^Y2GS4A9E7/s({tl̔}zx(|DeDel!bcTvjJ"YZ1iianEQTHB I_|Jy=ϻk9}k9\K[x02z? f3vv<n˕͵G`W_)pk#WnA}'Ppq9\W^b1AJG~eZr$C {./>sUX/s>gf9 ;*+?q09>u3C*]^@f6i}x^:wQ2~A{)y |݅ip!gϒ ::)A=9%qvGqx~(+ABT_g5oW/μѯ 5o^xnHtU-W,/*3:Sr3U)P/Pr@~U)Щ{{5Ӣ~(9iwQ&VꜝZ̅> 7OBLU ЕcrR4@nKUm?__ҊƏ"/Wm[?ieXԏ!lV e MSXj:<2F 5cqF/O)|wb~ 2"iJ,@; تP?5wۖOTz ΰyC_̙$);C?7 k!H"a;:Hɽ_de<+Y`]#yP) O)0[+e| =^(ܺ50 yD _;{snmTw&Z_KՁ U@Biόӱ?W!1.-{;c.C|_mσͱahJEoXk_i@eE5l$ՊwO롙+F+uKٳjJm$%Z okBuC' ug^Y\q5/Sҥr>$X|z0ҥ67i@H?HSXOCE\Myjzvrk)O7f|&<|d$޽C*5}n~@99|{o}]GP ߝ>=m̏n&0#܏*[^厢?MZ Z8/x}?rc@#?Vi^x|Lc`4ج#e@r?ҏ m"8F 6~~D3KbQfc0u*w@:\kڅmں |c{VOGs.3c&&̏\^Ձ{7d眄 n1Nk?^NMjIUx{&4^3aU[UTܮvyckbQ_<&0>dH$Γ:i;pi.G~kۜ=CɎY;c8`31K;A`u tv$GGB1ZJZ*/bAhll^a0r=0D۬:WYKlLrK7D@LZ#/Md)Cݲ .g"(3<+)WeC쾠;#7\0f6vIXRDMcQvwXד@_ tC 9'@eZ"^hF߰~*)"Ow7~fʔ~A^#!8q*u+jE ^D54iD4CbSUyF#`ۤ4"[{VIT'@~>8rOt$xo2 m|H=}ꐶ yN^(A ##6γ!6 Tht#&&_w!-;fڽO?m q9ѹV' wyM.a%{?:;k9ٕ=ŧrϼ}~Qoዽ+G4l c6\3Nw8l."ʹf>ȾET6|v\K^.=JG6%Myeէ~la.yV;'Rs 1a_b| gӼþ>׿q/ZC*]ޱRHGlѣ#LSD}B|VS"QˡEfX'눽bK_}/҆RbVtB!&'Y^@s(EK8z2̂f &i׼4v|9pYxr Q^U55h~)^P/rpj 3{K~ C^s܋PuA rRnGLYٖw:}8 ޽1yH*qɡ!BYTlro]zu%'0uN/rx AU%Иm}x) 2 H@iA=u7uڱG ܦ-O>松^{+)Y0m ?^hY SrqyK-[+a)Wn 93suՑ}T SzZ *6kη,&7+ p#Zij}nBgh9oyR}Ss,_;`ucAk, ۞|~GH?klTN"WFu&S9J5 .'c 1k?b7LraKByIXlQZ+^='7{y,(79;6Y5%|o:Έ֖񿲛6NgdCӓTSE84n&˷20>>eGݱ8[R-)9aOCQXqB(1SB-0ߛNZn·i,Z{m2 WeG;@_ҪWz >uEw5i̫0ΟⷡL-Pw5vہOL[ ^1QݾIcws?=45 vތ|;F?)4~{Y|8_],K4Vϩj /:}ѕOis!'X5ȕc !pQrĂ{oӥ&0dc`Զk0nL$SCpX qzqE~S)i؍㘩_F_\_*%_ ro/X1C˨2-g^/ qڦbbmeC֤EO>Z"Jz/Ĝ62g}|W^q-!K}1g9w(í|:o[a ζq''ǚz*s?75Pש;"Noefy%0[.@:qbQVdz9ڸD~Ǜ~ 3Ku-ofӽH,؄E/HX|mK=M_i9?׽| ~~Gm em/?iO՗ς,I{n:'&ԣ$~c siSE9^"PFkiꪟ,)Sjz$O5?Wy@le>3 Cm0 Ƙ'(-|̀Ү)J 638fV&oׂUVXkz}%1f@Xk*DL0sحMTqYfs 1"-8 ~',eF}"=+V(B,]C)1{ /iK> >(e 7$|=󸵾Xcڄ3S7dy51i񗳉exG8Dl&Pm>`un ` `̙ej3 pӖRecŖbl7cCGBVaO}őˬuJBu4 /^|5oc!总{mYG)3AnCKuVJN]m+[Ǵw;yjurrJcP%oY 7:Nc0b8DȘ@ @"jgYu1BE女Վף\7Rg :_ݳ“SBQ߅ `I,sS~c~Ǟ+ߕW.a,(aVJ? Am-9w-7NW )GYރ2$62Ϙ(uޮP8'"▊p؛fļ⺂08!>y%w>D3[=*>Eɽr Y&e<\|6-4W.܋@8;Ob!k;Ի'mW <av9z) j\|By hf'zS#PO 2mĜ*W֗fWݶL!A{Y.^# ,MGP/9E6$??mp83D{=Pnm*RT' Sj{n 7*.C{8gu\ţ}S"Lyb闄6Bܭcp1^S`Ԩ5}e yˇ:;7,ӎK?j ',җu]9XSnLsi}IXP䑕I;AA^og`=e>^ߑbrs #յ3ܳqh#{`ADu=3jpY 7;#UXDe[n߇`N~oƭ)> R$c\F]vw(~Jt$OCAGPÞl'G\8+A<E:Y-Qclh\g帴 =rwъsuo|i q~ުy{ryQ>be[soM8NmAuvGskѳ\܏wa[CsƻnWn|*WӰLp_|%O=J+]STe%꬘Sؿg7QwdHg֙kA$ǿgQmL+HzQys;5waveI\h_ 9Op;^.v!~$0wk k+}Or$Oޠrsa-&viw]Z*NgРvj5 n nƞw ayy^Z~Z)cV$ʜ ك9Sz<0N.{3X5K.)wGzt]ձe5}0y|HN?ӊQڪ\So8$6+{ׇsbx/- -!r5x7efcqbooW.q?Q?ZQm+`9ѼX'Ehs݄Nc9H ղÖd'^Wטنzvq@Zg +(7(rb$,tNYnݶ| -U. -A z6|r67{}$y"ؿoȋ1_%98.LC,}*рhS Fs$O!c{aa|9/b(EmRWb0bI[rΓIj/pa>ۆy[ޮoj]ݴ-π 5%󑁗s,QwȥcUp>DS0czU~SOSc]Yފa~%moN6A̗YYt ZY韺:Ma8_+dyMqIBXJ9p{HԠ_PBTKMϽJ|u|}=OsqמwFL6 X)PCpOv]Yĉ% ?K>؈}m#Fvd_7\πy}V{ sTU`/ҭ6ʝ)p?#BNSK޵ rC^4'~3v-+?JGlTj ^ϔ1w?Cҷ9qܷ…@ge\IҒZ*c3vri9M5)<>377^ĝ!x~#fD8=l5ߙ# /6%/pa[t6.V21z춵Ȼ4OWD8fz{ZıNiߌ2㏉O|m=qi[O66QX˓CX&M8~&Hw)#zɩo !޶y0Pz]*ѻ\*>x65gln'%? B:՗eM ~tq{/5} .Y i7`nNf% zW&Byo2oF\<l{J kT,BI' J" !Id.a%Z>7w!hϊ"& [xUP_K #7%P%T~Fp?uU8Hpv5Fkf[Ǖc60ҭ~فCYo):K.(ݿB5eQX,-gZAye/4 )JD ߶K+rMtoMQ"t^2OcW_4ro a%n=VS$(_ PB:7*/WSGYdA"?cNE<ȘpW(r c4c-Ot ^:%`3=.lC"煌A{,RM$yݼDY&9x8hjZ(t{Oܰhنq^v$ĉA}u{6p}WcHyoglq4֞, Xugmfi%ͭS@##y.įNdK[)ADgcߢ|]Q̬E P_A;bO؞<>b 3DzD}X^7~8i:(kgX ˾ΈNq9 ~fnNPO;-FxUft"=w䇑E?  :ur̭aYZu[˿FWyWT 3j6onM>Nl9߷ܒd9L}֖"mBحO=Ojm5EH/9uqكVm3IC5iRA_So _skӊ@i9 5?StB[ziQ bt 2O&ȖWS Re 3_L |p>FS rbc]6q2tm9˦|%[TAk(n3">_鄛zCya&=^jD.h\ڈist߈| On DZ`Y4Ai,>FkZ北CFO!A gH?#rsU$yxI)::1NrnA_t}x~On=FXCM&`k5ob 8FwY':]؉2m ae%^/g,lnwT[al`v}觓(kI:aL:Bv*ۑ(US^5W +Fw9'),~j]oIpwl/3=^k1>|w?J Zz,8Jb|=I~ j% MÈVzk|lrymvuu@pwC+m) xκӁ$,q7X XËX4~+ -+k$Z~ګawjo[,>Axk:!\B%}:![f!P <@h6ȃNPS3ϝ|Ehq3~kv@s*}RNvր*f9d{ɗHoz=t09v{GoqLMQ}b@?I"iav}[!(zV+U~ Q>Jww|͜r3[2C˴tQa)32jCIX|]ㅙp.ԏ2l{ e1+G`{v@og?ӘD&t Я  &w?] b*c^lbx>3WG8>@I؃_Hq[6:̴fx;:ҢiSv2=*v%jbK| ߕ v2m!x׮m IˑVro7lW/q!LV8 okʬ`D70n5͹Vi]T>HI3͢6@iXŷ9XI {4L4f?}+yq4p| ,/Ag`幇*Q`Q߰bVĎZY(Oڳ(- "(YsXe5c'%$l.0ޝ*/{m|3`ɍkY0 q?XD4Piae~$' ?j񁾳ofZm4HR`%| *^ _w=iR$ۼQJ6 45.u2B9CPÂW- xCӪaQ ,krs-ܠ=_ǓT,Yn15O݄wdn1LSσJޞP>eeJr~!n}) $@@t~(^ Ko?^݋wЁ7O^}no .|W,Y 9!~@8> bb?DjqE-b J:-|4 3̊h LL'o -^K.}k :UޓY-/yȽ4׫&:M9_f7pn9t`<'ift |*'KKp ;qF eEIw?ʟlE@߭D:{NӼ'DgntE%lLInX.s8_f7̤3}~upc; ~-AK6vgq=wb~i} Y5v*FW?bW%K:cjM"+mkLHjEwߘ88uHL_;kFDeȆ+pi.T-_䍘dV7@}-.>%t ϩD}P=[]ӱH9Wt_CG{&AjQ"u;,gfD5U F"vzt?e}u#"Io[yn}҇Vݟj4IZYk]_8様ۅ g3weD^뻦Ds`qrqSb:D%aͦQmdsj kC0I#*Cw.,͇Ihܯ#=pƼ5fsr~%-#-ϻw//+h,8kр;bjM|?ۋ^$}\W ; r}^}Mԇ= NP TRkytUop`_l" BcgXϡrCWCm&qYNi-K_4dΉd/«t*Uo j20Q$n-!Nj)s%{`4cM ]Xh"Y _t&SƖZH71r8!|#w-A"9J1Ⱦ[NV խfй_abɪ4;$6񂡈lo& d_bOX_MvDz؟.\]/wTkS%da\eĪ1D,thvAtk0H&>0 ֙3 ufVsYj-՟g˝0|~?VUv-a@֙c>I>$9/l: 'mDڄ+8c@o*P6ۏP$OYZS({O|[N=O #u483ߌтʞ*u ^z7# W7SbM< fAM ^3"vg:-Dz$hZrJ=??_v&'Sst cZW[-֝$OVt:Tup%um:ό˻ _77Aӯ6;GhyPi}CY;Bm7i)Wܡp5Z߿!rgo^Tg"P^mxvZ !߽ǿws+1l|A 6Yk]V Go'!`)[!lЧ"WGfT_(CO&qvby䇖5,}w7r6F k zd9ܩu^`u:nrxϒ V6>vK]݀!BO 8i NC)W8RBp}x@n)kou"9ķPW;W2e>'}2oBp5 P*xq;7'9 wzNs❠ t;G'I:҉wNtҝLx;WVng)䲄P2Tvp1Bo[̥÷nV(}tƣ=Iъ@=Db//U pf, o<^_Sn6Aawr2c:HmaN2v%PD\g-=$FJ7Evs'.}X 4S@]>YcnI~ Oi 6q=s+iW9xu~6pWG4~vlR!Ҷ#l_+Ѵ8qG`S;/ ՌLH4b<\ 쿃x$~A.] F<6➮[Osύvܞ^7 (LPJ{Dn[7>a{.M#hGg3G]D?|s ;Czy my=G4{֛@ӞM{i)$$JiZ%$!DSBI(”($!u%ц2%im;s8zs7&NQfO*4h_<5xz x6 :!0pwYjPV\Y5@onϴej3 n(yIO.WY=Y֛r,%z5pX7d=p Lywk{2ML*Y&sM6a2$ōws2Ҏ3~tCX%֤ sf?dlh 3# ֍T ٻ'!;lvFۮz Qzp/|ꑀ(;"];$S.kWbOQ&A>dtVtJ"Ŏ&5EzGUg?|_ tܠrgW9(!1I˸FG:.L@g~E)B^#qVD{ L=H"Ω/F&('LWAI~v!N[FuP{ 'Lǯ(M0- wVg=5:)q'yKtQ7]tL\bNWnO-'L!3փoX.EI*+zO[=D28!c''eP1L.r&:~ ]Q0S8{{l4xQDZG0#isT6jeڨ鋑sÆq~hZ2DF䕃ǵadBX0|z' ASQ"hWJO&Hjo(.#:2'v[_kŷncj}'5wnw ѽj.>^|zC]ˇm p` Zzs F\+~^Z&80c p@;}z$RHՀacStJ3Aupf dQƟɎQUc|ɶޮHpô/) Ĥw^h|PErweLδ-U&UXPQT 7~"l'п8wQ׿A5tЌAǶO_@wtJJ7U^},ݯxƕE)ṱ?3ʤioƽ"x| 鿩I~ص]ѺKv߁$,*RλfA4'.dD_|zdbO2wD_ !0YP?_btсXG'Ww)ZaXx'So40x?k S_/\Zrַ)2rd"Zo(l\w∍X3~H)qX\z* ~(fO2ai%B,: QÙR0/PYR_suyIudOV{__ w8~mlܴ0{5گ Q"vW>J:*_sesGU|e_ o3sĄئ> Y^Y}*5ʗ2EqCz[/;o)| *KkD5oXb$~F?PF Nog*7G-ՕvA2n|='{/m=WҸb¥6fx)5R"o ]\fRB߷ARҴD~u5Ԭ7/Z"[ԽlT>w[Ĩ򵭠 ܭlY%|^Njm3ZmlE;h#S[w&۠8VۙҴpX XU 9,/k$D^&BɅ>:ޅ,ǯv(Dz5bژ%ܵ(Fզ b<$Ϳ|l3v;PɟDY,! 9o93M:'ZEazФ=DfwH/wⓏ۞oI?K,_|n$N[x‹p'in_DFnN]7#7'w 2u3~Ζ> d̽|}`r  9?}%̰+ `9u|Lxz62#2P6vJxtQ-saP31! AmR0Fz-5@^_)>Bj&6Ν^YFr}Rϙ2pDGozϜ@}n1o0F_0TIl|:bR"껪Gfj|RD'p>+Jfy!ꧫoPV Vz'OG3m5% p;-)[=>k gĿJ~c'H4Q_Y w?iԏrՙxd7/:`[;L'i*S'ZSu&ܛiF:vh^ڟ rG#Ԭ6Cϸ{9.DH_qL;v㭣%n6✔]cOĬgZӵ#LnvOj y 䝝n>{Yg} I}\-L~dFD]|ıNn#?^l~!`j9})""W({t,ʘ3H5C2 Ҡ>m$4KȄ%"[IxvLTq/$ӄNطP-EK5Qj #/fs֥&ՄWwv,B/9dw{ޞľ.R(XwTw]AWp>`&,$|>M/l[u1@bD4!s?A=xx@"DXQMkvz.bLIlX(Ҽ:dn7Hn5r+,ǙP-GNf¾O7MD ILU5a}QEAʞ?!.&G-Dx|X1ܟ~HW9 9T'y'=T%>_!Ka%#q$x#jB5,"<2]U$< 5 B PN၈[W+2҈&H !H^?C ͢&p9Yv#t6eD%"R2 }pPQc}$a ' fsW΁mE['p Fo`y":[^J7"m"N; l5f~_!?ͱj&SS nh>CNgv'f`l]]LK0Cm3Ȥ` Fp^$HyW&+ ,0c&o_ }N^4BDG 9͏'>ؿ =(HJ+#"J=ПKcQ?]eDa u-՟moo~gO ?9np @hFs;3J,- DT֬s?]NvQug*`FmǖL1-woV{pyqʸa.J>drF!rn:3}}^|a_u|_=g:OQ75'&( pF^ym3uGɯQGkdՠض^BѪo0l+ڭHGv j5,D1;uPK͡_N ga)!TZYp~-E#Xk=!n '^!¡r߱rnc o',r> 1?o̲}A첈qwNe6' mg: m*-l@!y:"pAqO(湲Zi[ң(Cb$COr=Wk')h+ǭAN^o, s]aPKDFpK"LGrgaDPĮc=t[X@8G!1}՟m('a*xK3a6H#uzg0c{J~Ra r}Y6]G.~pUk4p8Q♧l*alY?ɣcܴlsh,ύ(!U!N;JA~թLy(6VBKCZ7Vh'u|0ҫ`?>%t| ?Oܶ_,H_]OXeb]X;ud`$ƬK;0eY Qv_?idO<= 5QgCmI&#t\ OMiUDYx"ʿŶ-DcLބa*GhTm:ĢsaU$7hS$T҇0\ 5*(}u-DQ^+BƋUmi }R:\"&~D Rw$V'@yf_Y:?^I1B."!;g'TW1{%ȾZ 8p=394NӅ!_$p|#5}8hO1y+V/8YGH;Wy} %\%euE 1gYtB@;1w о[+T{iLzQ󵂷`im.֥JZ"X+\3bWmRBxj9' cg'9W>UNqPn[.( }rK"&<Ew_Jx{{}uT-s V4{A*׬n@I5 #(vf}োz6j"-?=_Myd41M3܃ ƛx7wB%vO7ϜN6gE*_EV#w. [շ:H`ˋa<2>1G#@˟|3&ۚV~BMߺ܅>9lpK^rUu!?˂#Cs5tGm!מ\qߪSQ& +[I>WW"tJo0ݾOD(oO5##xz(<>_o7BEkLn|vC+YAe'!k~W -E}gs"53>sE2$<(/zM6F/%Ͼ!m2Nm/'s}Q3ZL<$h<7`gdIH %XnyY*JMȻ RlP}]1ss ӷPLPXXUuZPdζbGCWQnW(蘡.gPug^Ő24d;&ꆨn7~5E?i=lTjٺ!ÃhBzWYo2F2GL'o;5_OE.iLYQτuyrTM = ̣$zUV)htkڷnP?m}|yn%þ&7&ZօؖMPk'VhB"a+J T'7ԇA,[>(9aW\+C e[N]m튈Lѿ >j"Үy- EUQOOk e+vr9P Mqi;&x=g.nřK2v.07G̻gӟX(Êɜ5i{KU&e&9gRAp|datc" $mc?ы-38ѓAȭ}S7 .=SQaLv}ş*͠V_:v*ٺcɠGW{tA_=eYUdVT@L=Jp|ߞgi]h_otj5"FMj>C_ Y6\n&8LO78u~ oyL};yh|Tϖ2{;][K ){1 `;1ov8cO.%%򔾵ìo2V "U"[Xbn[S(uFoK 8yYe_76J/8K+O%3+*VtpQ ~r'uNY2~B7%}p2K*Zʷor՗cĜB NVݪ,5 VS AHj L+r4҄F6~ƞ.!j #kfdGv6_-Xo;Ճ&QU-?MDMl<ָGO(ϰ5kg^6_o9ܵ+=lCWL^K@w,t%>a#}-k%tw/DK&,0ige6uʟV[*GX~T.U;Fe~ ?-4/`[Õ8Erf(?qMָ@8>Q`/~U~565mze[)jR56$r| >yT׹@Xv]u N,{qF?vO-r^&@t?Ol.>rOd#%_ =e⧿_ g[L;q`"䲥_NAڴZjUD+2džvv nWs[ۨOEϱ'dX:wpG_–mBq(I![9uݳV52[懣?CtA=="H!R $CR i15H!176VBr FH ]6H!5Cb@ i0qI-$gHCi B.| ߛpiRp77!*v̀R'~W0Kf0]i,Ջo{-jRtIq= {ω|` w|Pn.W>HC f!e{I| jq %L<(֖QaF[-թ3R&7,3 aުca-Nﭤ/ ?9>2'Њ~StO+t. NU$o%u_G@_x y\_xzПxqHr~aCS.7kxe,JoӸ%Ъ\ 1)XVfqܮk>0mgE(NowP dx[(_~wMEC8gݺf> &ل- ;/ bS`|6+˾B޸ r#}T!QM}{qП9#76l㺲8`>n̚_ȔK '4Ǎwہ/lue S4V;Yl MsɢW^CgR'' Ϡ~h}sR b<_Eѯ>>K+?4!ӥNnޛ[y [cL}?h=IQF! P_dXW|E;:{]7]=eWψ1>IJ|9TU6F/$ٟ;iǟbG؞-[{54Mmvc^a՟p~ZιS~5j }\Qڊ~Jcceߌ~;+ ;OvS ͧm~/"}+\q`5 \ܢj'6f-%{q6(u 0NJq?ȌX`3D6&2˧"Vt9d{Mdo06g?v3z݊q2Z-qQ)xCdBRvnsObN{$2+dSf=|S^s}%蔃mNNo]|e+e{ 6} xKVAO\ \FfJ=T$3Ʒ+>2=]P _r-L^ank,މ>IyzmE^B>e_GОٿ? "QpL&L!Nݩ9?N`sGQ_2MdF_k~6o1mcCG>lKUgo+sQqzNx/."pd| |aGeg_t^OσܦcGG{~!|݋:3ʌ@߼DܹmKM⸓SJMX]M#~l2ǣn%H[«ˢ(W%=ӽ7Jkv>-E8de.9¸o:QLڮuZJ8ꮽݶ, ;opu->K}g띶u5`LkK.pEY;4`D"6^L_& cM+&<_pI`$-Vpx]$pF:G"Ԉ|7dPQSWjjI%\X#Xc8sR>3!kCE>-̜W~gIl9yQ+p碪S=~ ~̼_ߏ_?g1XQF&P/ Ό5~*wǩ=w7&5GL /{Wo~]?|xKErL3 2{)r/N&Vf8;uݳ6?fy,ދOY([,̟vl7QJN۠FW|N n(I闱-ʘׯzxh_]J_ oG{EI! i}s%\1O%`IJY9V|`+T'IzҪ4IZg3qz\˚sq)$YuKj6sD1fΚTeZPO3"Ho4&@ˀӀ/6qLE/YԵ$j߰~x;/i6`|Ǟyz|yP\I`@{ZĦϱ'uV{F/$tɍP-B2xt΋R_DKzww5 덾H8>qq$ͱU{^UҟwIqP:݂"FJ>C~ӓeYoRo>Dc?2z>wObۇ\'eAw3ϐF1y!foV2ki;[0EAy3fZfݷ͑GjN q93Q9k&~.]^~g gwnW+vnf`R˾:O(&5_vaO4zgr[w6X9zOҵpXk#ϐkL|Y=Y @G =l+t=ѵA`OqҠ/T9jYP_Dh2SRm JX{}iGECy#-_1uzE2ϑf}"4W>KztURw/JM >!>nEc]5~oz4OT5ZG[7x! - 3<4l879|jRyplQYoƥ&{͔?'Z]m%U5S'ld=ǵ\%?㻔` proY3qw%nID|F|8$"~Za<ۄQ͓{nrM^fC)l/GS] X>:^ͮoPo _Y+`pl#RnT^ʱ3$쑥s1t8 ,ǠdT"{ƃVۂvY\_u3~G6)qbA|q{]{]:q#v/ɴ仓o|u5W7^TXn$wTe/Q\/dhā*΍^{R& rڂ NBLCaOEgr'h-oh/}rzB(6>)0q}RD|?$XĖHq\|xo<\v{I XyA`j>oJ0Q5wEao#tJ0JܕSث+y\\`qghb`|}0-?ȣ*wv`uJ]ddj/:sL$"ᛤt}/$*"QQ0Kaa#nLsf?{194s9ݫ8F|tp>6=LOóXӾ&u+O&b7ixxEǮ7=bob8'q ``WSG :[.4 j{x)/C$ i&fGy^׻+ GzK>WzZ3?b'|ɼ'v:'~,|IJ‽0_N{@^sц: yݶ)uG?gEqY4{Z8z?J^hgiQCODH|}+z'>ZTJ.&^?}. rmVO+opl@E{42ky }<΅){ݶy/)#TŋYZ;F,\SxAXYrԑ1SlsR,"pcq1W>J SRՒ@޿dC5awD|+ ,ͺ硔nbjBypSE=-$4{q7҅%Eؑ7/@?͓7#˷n61rRHėXWO˾|+o$?vWft'x;mdk#%O-O\Xbu ow@werfǟe-ƝwMjώSdx1ذkG 5'9pER3OJ ?ì![^^4wбtKj'zNs9O@}:/!3z(f}R;p62eѲ!/jz{N&?iJ{ 9 Yu]:sD:Yz?b$.o syggz{BOp~WufVg/C*n(ٜ_'@(%k i;Cm0T$*26-DIST*TIHERTR;},s=\,BWvnp&i7WmF~O~Z&c"kZIS壌Ed"EGC=1/-/t3C'䷙o>`=BclW.<(b C? I͍p?9i䒴!!b}A|IjxT} M/0o=-mӶjUp]ϲxy _rpKIo[5 ,5 xQUk*򹹮#n8rfxg /R/e{ySlzjYVوCװR˿3u辉~4W֤ߒ5gL"RT1Ĥ$so1.w_:1kSwף S#qQTyٶ5HɥLzF7b?A2U.(r~c 1 *_t6r\n*4Dϕ|]0[T\D|J{$T-ӆe*<?T/@dž?#_0#1| 8Ɛ_ڟEܟ M'fu@;'Ϝ*O"I$}'if#rU"'[.xF5!KpJ~?33|9mM <峉ZߥӨd #K_S~9BSш#(Zif~9kM/E4LGLK9F|AQ%`@3CCt%e o˵\qU1ڞ/jyF&K rl$/ݏ牠_P޹^"4)xP$(hA9@ 0\Q&FF-6M_K' NǪIҋ\sZE2AZwA P&WXs(Cks6iʘ*rqY-m[W h:}gf [֮Bn-sԛD#0=lLh:%џk&ׇPs(<֧Z(č|t˺gȄ,O](/ay!RrdPRkWb]|֣b#~ڣ֪Htu3{<`j?'I\=znJj[-f Z}>f`,vF+"nhO'yׄ#>9%U}/I0/ދHX"4t0?(D+qx +} *j|0(PAAW,He MOlxJ^QaO_3{n 1fsN{)D7/+>9̩oN}sSK>:ԩϯN}uS,a3z~*V |o>V?BmGO؊mh3۶P3Nw[GENG,/e*K\`17+μx[9d>ks\׬x>_\Zg,A7%ǩ65V xi _M^KH4pְ%PֆS|ŏ8ުوX>XB^ uGXwfJ?pK-!O?1y@{%5i!nSJ{g }!.Z6 K!v|1E%1q)3gDTŽ!.WY:q̑}?}QkO!qb:ڧ5[P.]q@=zC|4H8?=4_ _y#,~+5عQ13$|·j9.#_m{bQ;#կ߻_}Ŝ!yWgnYo-ߛ\&w~_GZ!X_".\Y-m@9b{W#d(!fʃ&b0eù'3Q_N3/O k1Xj'bRG7s587}?+-A! Gs7FČX.у3?@Ym*89ƻWKy 1i!OI~ .21'pAb3ȿIfic<=YOB 2+&|{O1t /'w?]l]F 6/5Hw2hbU\X9|qes"y;-(Zr։Q+a5XE9ӂUS_~VXYAǎP^W]-FOk.Z&W\T= !YK䷳d{XCb@9u㎆5tTo>JeP+HVN V=BvXM혧 7J 7f39g1Tvlcwˣ7Ew-S9OE?Z@uIY _QȊd~1P:c}v3MkxL:սJzߧƸ>gij4H3\A,Ss]|a29S -Zm?M( ОMW͠„;Amȕ4fqƳ֓_[ߑ̠!Ay~M }eC׍M n8޾;ۜ-9*58;з-&օ+4-Ӯe8B"-8zUVO-gVJ7jc|Z6z#и}İޱ5Z¨d OE?|N%lSC~ ɏ߭t:.o1Gb9kk x7$M^m~TO$x̤[JQ9X^ѵj-y_n'l\zzr'csP%E *̀|dݹfjK2MNPˢL:2ɶ|-59ҵؿXp7O;JyCɬV,CspS1ƌ m[>SAJc\({y{j@gEBw/NZk¢9پ]U?Hʵ}'c;oa?O</?[Q@XRc- d aj/_9_hUdQKwP ~?rvy}yjA.sz!MցC&KR|K{yigzpN4*m> PpNz=2_A3WWc(P]C~Q=@tD%Xj/.R:d bW\ 5NwC %;GoP1VvoE |Ao` n5__lتDuQ,ؿr+w&}h҃ygDmW n~Xxv|oMxk@dz '{˧s ͈˥~[Cs{V׼@;ƣy?yEoXt B^1&"UU=g7:x[F3s᭧>+az)0V%@{PsAnͻocg]vƣ䴇:2jvay /I},u2*k~AVXADZk de7p>T@L66 ^$%); am0H&^c@4fpIPSBw no*z|+|<bkY+r7N>L@ݽ"_U8Wd^_0[L[#e}HB%h:#G.o$WmR|;hC,/,~kԱZd[3u&aY(dL{i&-TXگ8/48͵i* ˛w?X9 c*'w8s)#ߨ,z1gMCyM V5?Fn&P;TK9jzdcN&mZH3Tϫ"cq-5Z4T~~9%*4tN T[Q]~Ij'imt!p &F!5OyBzŲij ˻L;E+Oާ8 ZPZd7cЧ0Mܟ/:0Oѝ{F?$i5ݣwCÌuggـyDږyXվ,`VrlG(uYk9mv0tb4;/2%u);C0t;2RZ Nkfqy8 ̵;f8JW4t]yWh܎'}! q|t)brv`Rva ]mwou-R+'xiIJR/+_ :DԴw0v+i)؄ bNu ``i NUZeZ /MJ]~ E!% T[YRfKF96nn7XDv3̓:@bTb[>,zЊL;T Y ku7$vgZ)ND+bߥ{]Cףw>7<p=U䪿nE֜{2ytI_.õ[Jn8MYѽ 4.%C ;j5 |Bkhߐ I.HsBѸD5`;BUkieHK]9m;y{weq̯;f#0T(MVAδ/cSn5$wً`l`Ͻd<_[}ϾtwB@<䨭Ҟej,Y؜8Nf8`N\d_blz$;C\;Ï%y8F#&#wK{0_.sCns{ДKMw]'=WlvaZE.)Ǽ Q^m/uX g'\1.,Fӗ,ɼ&fȗC&~9z}WTہayknPYq׼Fr]UvL=;?sz-c^>w(*0, 1c@qx|fdj+(>?VY,Zf%=h_񜴏Ӵp'ߠ!} O*mq+iPI 31Q]A4`4$vZ=? PC,O(Jj4h}Y'h&Ȯ%^]7*FScO%.+N=1ጧLB}8]\P* [gV6{yT(⒌~9w8*[w±P'ps7_Ο5޹֐#*`< dRbnK䵂McH?y<%8^?i>2 YBfa jV5}%4z6I0Ek/9ɽP72 5B[煼)qV NtFψKf;$\.H+[췈U߯ZA%'n yM{kK<wJ)o_iC_ܼ ȇvoҬ7ܠtk‹װ?Ski۪Z/Yf&AYe:dha=0 wV sBڐJ=M޳kD_o  7k:UAŴ]z>ۣI~7{1,IP0^Q2%}[]J]*Yhrs \ØpY*9+zcS!zpqf̬4"xB\j(|~ er\ 1nF0xOj8 ncVةֿwve8S]~UQcĒH޿.(UZ# @4Kstd8/vPwD,mؖ / _5hicU~:E6d|~Em}<6/'`]PNbhCDuynNӁmㅉKΜg 7vEҁ. q3' :; /;#IGy;C;eqc>:loB։3^gG }|(_/F2+gN|:+G`{kY`Nlq_F u5>?]sD#nl-\/%;D}M:``IODVҁ"7~4ti^ym$u x?gu\|?\Z~ʕ ۅ_jނs8M8 w; W&-Y M`*)`} u7CQ=j%>lO _u83@!?}8&K+yxY<侽/Ǧ/#ouR5(A>N;mcs旮X'Eأ+2JSu's}Wb?~M+|/"<('? sq  y6mC('DIWşD~bud˱[{{opX+XQOTC 16>O&+ր\l'`?YR 9g`OYͣ EW'F}Z>VB뺖\zD}zcyN9n@FtժX/8TS+pMK|ա)o̘ndbn4=2Or 4''iQ؁QzhDWx5Q%V߷ͥЂGL`Wv$[g,s=U/ooeyޖ !--!0J3f- vӪEU{c_NmosxNu>y#c:EEOFٟI C@jߥDϡHH߹ڣ EG®|%XӭsXuuAӻo.7˝ͥ|0Y[hƋ49MȻh?=:iŃ^ t׬AZ]qbn\DkM[O-)ƿ+b(/XGGکjW;7Kn} 9 U[l ̒EYX/.p_)o^κGxd=y#R&3)4+ [f}.YHM۬}*XzC̟2Cݫ.Վ :Y־HD)6A SxX/D"4e.M+yÌHO98wM$Q &-$Ʈsl l.MۚcW\@7m5 L E# : <0}&iY::f|kX)dBz\Dk+b#o4'Lѳ}C*C|Nbt:m{{`4k`򾦞8ʵf#nO|e5R߼%t L 2vk"n+%<&p?n1 <M`Ұt/gNugO`~ę$ZQIJJRgL`9k QiM?S6x1idL 8~_3J!H+ewBmFl0uj;?4T8eI>.tcM3j#]R.@w*~4#Sdpه`J}ayMPJ_CE;<5ߎBw~$ Bݼ uorHt$>fL~߹$/!Xթ F UNW;X8k@n˓`1Q!YGNG%vPyGwp:.`a5ϳ=69\*{|9K3Wv!i$~?O~Sݟzo?ečr lZ*8 5y;ݜ5O[l_@Bz#xܾLmB7\?.]<p_e % {tSADn~*QA?bĞ  1;.tG}_¤$RJkc(JU-wwN ~v4/||trGϘqW].uJμlnT}%c0W96JQĞF6?d[wI;B Z% us!92!`4<bps C@A_?x򌚾:jC`h}Ɔ2N!! kDWG oPs%-(syneEsdMg_|E~aGV08;prɅ[N +tO;_Vybʮ!vm~^CG~-c8e'Ql 6KEo 39@@y@RXear[6Q.qjo#Աܷ#=cj,X<_S׏}C7\_DZ%cYH˥[.CR :}:S]k_f]F9xz#vWF-H^'c?۫SO6Pإw/ nA;*pףO$g]OoZ @ٞАBwQ Q_HbO\hPH6XAV!թ]ұMCI}<_/Y G`Znޯ_lD$y=40YyC.ͦgL(5Z `0K&t.m.GJLWLȉcQAA}o6I^;MUsag+[bj`e' F+Uӕ@hZ!} 2z zxw2'?zm&Hf4G{Ud+4̫s/UC3yY{Y\-~Hrkd=A>qKO,}/?O,t(ߌŚ0Gq >235}Рߨͷ AGZd!oY-C]iL(z)ΏQ輅 ui!};K+θK`q`2=`HCB"Y~xu$3 9.ҠɫEhrÞs/:WseBL{a7q=SDXw]'Tz0R?8⃾Xݚ5"іI)/ϒd0LOk--"%C}U;?u`]m oƯ[Ŕ#^0h\g/zQY5ߐUR^ͼ}xSLjH[.o-G7L~%JsP3S'OJa<7A+eUg_w; fc30k]wbZh=62Ob~}MzRvp6_u64WzvPkӨ+a>Wn{a~"̗2'}euTxWsíW}()V:L_y7TIrzҷ_!f^|GNT%bv|ek>ގr쁮" }E[,{ƄuVD.bBe{8 {lu`|RqHq}dP% vm&cPi3qGƉу1)㘯:`J}@CH>/!z)+k߳1G~ݦpSsar:# wVRoZv ) 䯰 ϩ5'r{$cӘ(;?GqV7Q/iͳX_JYr^?D~M1^aF%o+0: kD# ~GѠYӗ8~b $*t]xm7Z˙ ZOv|{N+)t&$xn]n256KYAVm7`\.0jA)nuwweDxBCP{Zgd9 _c|Zhjծ_CZhXN>'6Ym-#igo`1-# zS= Uz[ chAV! փ霏hLZf Q2f#uGq?Ѷ ߬|[TgG{[_yE>e)_z^Q)oP3n᳆GC^l=n/}|Qjb%~ qa/q8~rmr7u"~{9qż-+oY'wXLFx|^Q^z. *%7n-B-„s_*]܋+ClN0ξ8nR&7P[Wd;HDict5ng&I9<+/-kF?/Rd Z0{HmM̶"Ɲ I&~G|Fs L^~]C$n'F;\HZeb8W'_P K1Қ]nG3Kz)"qwDPH-tįƤ?Ss]k$ڕi`/折3㹏 U˷;*^K!*[nSI)?v~1fs5u? r1B_w[lNx>[&"4R^1ōO:7UWĈ^s6'I*ā-f@yV8g6 7cC)2oA} 1:S^Պ=oe#/0l r~ ,f[/:mӺ|gay_13=I4|Pp٘`i008P=} j\$6cbyԜvnr=m{%gOr5-zTWvr w\JF^(C^`!򿽺.]>] #{%.EZE|6LD/RetEo׹*xbyCi:sM@~1;! eZj9]?ǧ&/ {TLJd6_Dܶ]lqX}͕" @|m{v2HHK񂢘p>f l/ |(QLxPylIJyVsP}书F2ty 2F_DV-BQڔeǬc'f(9sE{-Zq6ZIz嵾@pƫt(>QV8 ˌq}ɷ#.^㣻92̫=6kzM ȼw d;{[_XtacDxO[, bfx=m7-DL:D}6;ų=4s.U 6{!>%zV6Kk:`$3L26 ;s/c8R^:ݨ\ v4#{W jBk^J@ %˭-hCx HԀGlAzvw8q-3}HF_vDG2! z$| T،C|T<+Tܹ/1Q^&Wj0܃q7ԟnz}5rU=5}.z{eP\@ !n_T/ yB_*G[W."-"~Ԁ'gMxS5 #xԠ)KpAі@<,hm?/fvt?Q?_,ւyO҈д3jyncDtxU=n"P /~i-  /`9ڤ ymT!~~G rZnoa/򼇦Q)Z99@(bWjZ'mm9j}؇93ڳ6+,j񻜮[k dp /PX8"gtuގ{(/}.)]f`/ŃT,Յ/'^,5}赨%M!Im/7s3xh9PwytԚI{:Z^]V[>DtGjmw_ʷ Y7.tTB.;8[Fۛz]ܘ_ta~-Z{b_s:DzA-.=&Cʳ XJE Tf#oS4K/tÀg< H3yb,oaꌵU3X|nɑiTcoE|N{/9oF!fѡ(HNK* l!cNY}2l~='oM|"S_[&#: JH&fEK ʩ GeKߤj4և'#E_sIƨl=xQQjw/"~_n?XuX _RZ.8`W'EIaiXTtɾUH֡$tHQ?1b|(@Ab<sjm6bSYLTƲ|==>#rEʖy)m>poBzk)O>6 zeO%DHHO}7́p$_ЇÜhϒ6%erQC>촇蚏V5?qELܰB}˰kS/x] IJ|>sqV[)"?s]ue:zآ}Z2):DŽ YGfNkov][Ad?Wa?Ov!i(Ĕ>m.pok.o tsݎ'Hg->8^tuby'S;,~3 R>G@4U!A%>͡225w-/ {m)c(N *d?Bnv_[Ylp< $BEP(.;x\?[U}~θPWˑ>_:5"T`ϼv)FUs!R_?F7㥻SCC!izwx "Ec໥ع5l3,;}沌D~!g^q-[>7kUCC`>&ЧLx?bFQ$3#;Cr7M٣? H2kI9wVn[w@m=Lߝ S"y6} f@\g`lI.ۡb?kCns #ء(/ H`?!0Y'o~n(߈lRl[XMsO&yfbQ-Z+|y>[l!kp]&bFf &_V 8Ij迲uI4Ν̈́{4`%=Кof4hK(#Ȍ4Qu6IߊCPؼٖv1\OvtgpP*GҒ0Jh #i/h E| /INj8`fƏ4qV Q{ s8*CՌ α7ق @)Rg 13c~X+o _1(/ssZG]"?뽺W-PfvP.?;1'0q}ңjB,leKV5czxΩ1#X(3DM,n?,/오C!0ycC46Q>*SZeN_sBmy%i'fcRiP!y9Km竤!iac~$TKIP0?z]tyQg5>/+pj3Gۘn r_#^F=m9? #Yk|Ҳh >SC}Tf^FQ$Ǎ[A߰t~{1& jс`Mz/%.Ž B%8[%~"82KXk̐(m^xMɽu&WD~OOWTi} [|ſWS[cpӁ qN[Ycٛx߭9E.4gɫkhNHk}si3)LPdW;(,fy0tȷΐnXrqxT4MsKOVy@#`:2AΠ]>-xp] p!~y5jםfg M/M;ȅfȯx犰 Ph5:^ЯU9IV{ʉjܼ-e>^mߘy}(iS.>i7N Cu|^KW|JO^.b@Tp fގNB'P;c3yÐC n߀C+I'\C7=\V0UoK[l=W-~5ډ)>P\ߖXKc>0WϽOwF^ϠC) P*ɿ: sV 57:nxSLÊt-n ے4G|s'fG]n_ f9mow\$9@[J#Q`Jo`?PwF>',ܑxɣ/uK?e4=g^jɨ˸~]QO䬽1Tcɇiglj8PQ ^9mjdQއ;ˑH';)?<#j+Mx3#S!*rvv`9ΟerHNr:dМܡv3.s> ?9D^v %8e#g-4݅AucG63nHur}G*e#+.UЁYPNiv6Gwq,KL Otⷫ½tݦ؊{}0Î /j|vw|t(i9;W;Ek{q t(Zq#WLyz.ǽ|e,UHo69ҿ=N^7Lb4 3I7 Ha׋u|V nKP\Dm 8| 7g=w5(qB9&(8qnq҆.3_b1Ķ;AJ{-(> zX>N@F'` ~RU^όuwu5WHm(TgX:[8{,PYf菚G܀N.tDkN3܁$6a&w h_a w#akpu}RElkFڎuOwxϟz_+_/Zwb}L=1}nK2e7m:1u8j ]BO}+ieV+=P~eC} [cL"M۽X2%`:w߲iC݁ntZnGM+@,P.3Rˮ64 oT0eK۝`U<_@fv]͋1IY11=X>S`7@ Uy^ VNc:#^z4@s6u4/ѣw{גSs@m9A w:@`߭c̝]s~w[ڥ^\@y͞v4嬟Dl_>-/ԎزG)gQg9؞4g8.FT|{֟w^Bg!]Ń9nsL,r.eUG![kOL0zu E||}>xTl"d[>t.b9PJ}#V X`{B|wJG*$tGyf; ]T#?vOq=tCB2k8s!X6E{7li/FFh#s#fdy@#vHEolL-ȺWz.i#|coupB}IN4D=?[4ЌADvja 2"sK_>-XI]{{b U<7@!q h>R<`m2MC.a;z|)Ÿro$V'~${t<[I'~d"OV³4 \ژJAp*Mo߆Ӏy~AV?=H]i !M :GhpOM(a m*svakweTm[_I m6Uz6(b d#y~iG _y}/ [-Glÿ_y֓NZׂbF+v;A؜vMiUp7Ν 𝒪wl+~20lHyZ3myMZuꛕJzUZUTݫRz֑ua }_NpymVhJ.MR 8Gt(I[40cvgfBgg:;6eK[Te2R 0y9J#\ꃘ\={hP]%5`Zd4 y؞O XOEmwVJļ Gdw}*=q5ezNQi'[-6v$Y/rfM]gCYvx5_k{^R:r\*sh?޳͎vZyox*?҅Q}cag(~uJJ$Ls:='kN]g+:Sz܌qy+y闲\{HɨПIC߼t0tKBE6 ϕ{%.$ōZ?^up6HzAK|fFIkߙW?^rÔhE(*8KO P+N :`t%PN=m:y7=LpUDӭ+V-y]Y:h#3+:'}Gk6彳_tvi5^} ҫ՝ޟ~0 AZ7횖<iK׽,=3ȴr bB2Ths!vGi|HvT;Cns^ %>v-@*G5I= W$|KoWZ4rHk=dr@K\Fѧ@yJv%"{) =g[g-VTq$UڧҊ`\TN˅'VnV6}^C&GLo W^2Z;4]r\j$L K|BsiEj/kzBb\c+=iEr/ٟ&ȓ}Q+[8$yJ[af cWΣ3 /W?ng#p*s,9랥o?Sꗿ?}5ͻv!cm2;n@~1ߡ MOBN=GRqmCx~fh9oI{8[kҶ3︌uB`űCdž$ 3'f@isqZw_|Wl=Y[dmC*DsN C 6DR!U[HmqNSL|s&=7+;pg mm]\Q濍d tVEõ&0t]ຖHMjv]okx~a!ƣ&]_f)C_3[ўJCs?1EB|!/mϭbԍ>xZY? fhoV6IkdV`MZ  ]oA:7eƠ1Yދ6SG:Sa Y?eDw+<ʛ{bاgN |x 0vqpWRD_ kp8_\AW67)3Jn Y\C\]%a~F&av8֋9ICW췖.}.i M3P~l*oS`PxfDWS~}}Ek՚F5IlIEu2G' oeUDc}Zoë+Nb>|PtvZ%svї9dJ#jE?;6#I#F@4"bbfn jHۚ?3Mn@ըA|6"&'aۅc&ǶXc(SưzvfS،2:aN-1L>JL% `h٢>vWyaDRݕKfaki\tLӏ:'IK{>R8,x24qy8jR}-lVi8'P޵VԻѺ[*.<;XЊ儽$wy\ G+-W48*S;Li7K٩hNSQkl~o6ӝs}y  uR_zn]RFxL cqFL+w}0I9c2UsrӿϽ8h ֻ-3k3`brMs.6;Y ܤjdVR-wcm8xi‚P ieu _1K]v]Ya}d}` .cۀz\sM ߽]'(wXVўQe.HWjAhh965y#a:{[~6/1IGX{{gßuDr=C ^04M~bM1DRH<3@Wf}5hѥ%8_ ~f@xx]!ݎCc:W-HA[:x& XB7d+>ckpj֐EEP!뎺]+̢Hf~v$׫NCjUVQclh CWa}\e:53WqJ,6N%E*<_IcY*ԂO9fPvW q2;0so[0>O6w!U>~8by112ެu+ȃE0ľCP?~]F pN D#政5)GF;c7 ym5~l%G HW⯆@ttt<؞6`>- ~ >s a5qUpy__``ᕡr6zLeasPC,'@>S 5f҉/-u0n(GޞOhVI|G5 }Hm[}:[2yl ]gO֮PW9[!Sz(YgfU~cS#Uh+\룃`BuctdlGػ\^_ZQ^X۶eYe'K4ACOD h~jf=5#*QbhzKĬhf H2UH3!w{ۧb_4?Ax뇘tН*~L "ě]'Dvϟu:$OD4"&GҶ1:,V?^3`Q{xYb_fv/Vz5{~fHM!e ~!\: qeFZZT |܀)t@ls_ObHb?^UHtċwKK!|Ő'YqTH7 #kRmjOU':b\XQ0y*Ai>skSLɯ"&"N'(RNZ/D?r5bdmEj/JB܋uᜧ"3fck"5bKyW 0\o97wmsWׅG FVy"P` [m刦O,;QOXD~ -sP2-!n*8_ 2sZd_Mf+Y/'rIR;Z(,s| Iu ce I YKtb=hXIZ:3 {b XX] Rĉ?b[)NpNewMgiXWnFXv0O}b,LAuv,?L)cG2U2b9Hme$r?;~ډccv(x&+|֔;툙6U)1 .MW0k;319m-帼H*c8گu͟Ywc"MDڅg_ cB⹆3~U ܤɵE|=Wn Wסf2bxEаޙt ηřbt`l[ fS@~m5v5GȪ6v1hp2_[Jk2PQ#"8txN.+YbIџ6on{:KV KUqJO!VU1"7eE۾/ ð%dɶF~ם>[B1om#p݊tr^ꓝԀ%*j ȟ|v rEh9B\"ؓvi#l{_[$e%֝5#>k{mج%G& εfe =OîBق8%ctI d֊:o V.2%Tr|ٓMfϛg}IQ娜%Hݓp8Vg}}Q?ʵFhOi"{pg4SF 3aW)ZN?]i3v|2Dz =MߐW{s&XOxv6s]BvY~ oh#J?ߦFW)iiBѡ3Dx1IG),c_;õJJo}:LGj1 5Dhzpkm 46ɵsϝ3?%aololOMTef(Y'EWOʲerªt  #"<|eQ겗0SANxM8ga17?f<鶵9g I≠4 [/%SbjڎNa4i~/ȃ}OtlG+6sgUVzQvaWN~:1_BkӁ}VgWKsRu fU'`:&9f7= =m*. ].!Tن8d#1=Bm\ܴm1bh WpOݨX[3g0S(+߰kݏuU*D9 _ĈW~k=H%*vtbniLb/>[l#ߓ7#8⽧؏9&wCm ^;lA$Δ_۱Ly@bL2^:N 8}!4 Jw;\4}|2Gq%.٪;6^KL nq$_| D3Mˍ4ǐۃ[eQ720kXOy8R t@?'=k4 39^xփ24ssDDސp Iz( T1:Ëea"難e q|SH `'$=?Yx M& èp26>f DW혔}whMzWe<0!mjH 1s?yIqsummՆ riwyٞ}:X_Nj|]t,g\ 㳪u6S{%28#iP)Yu-3G8^~KB#".a7^䈠 oS7zވQȽR= [r_0F3{0OMTwXC PD#Wu)_b|P$ _{fs5mN{s X44[ +˃!}y=]yZvE~)7jydϷf֯R@W4W?Gzw*A[ي(ϑ7?9$Ҥ_J8|g)杰XfzM",yUw?׮3 6 :být  ĺ 1&a+|?k\wFodE2&T%"mw`jޞrDޖ5C\ Pn3Zd>2EᦌX;~񙆵1pN~?uZnnm}Wo[[ Ztq3"+|ndh8lI$^5žng}WaQb^H?22i6CK`WwoEO|mg"7XhD̹5uy*@}~6bՔ奠bnbo乏3e8~~V-sofAnv|#گQ߻äI{fW^){ce:oQWKK8~]JbE[ ꌀX~07fM#PuYȏ};!v)tR{8{(׺i8~jcf`oQ֚jc Mu? a>/Zn;׿u:ZLc`]yV>^i f"fH풷kN*R@:ߧO=e uw^XoxW"[얈 HTULjR0O/߉)DLk㞝qOM@8F=V"H oh넸P֟9 /4>2HV6"(k٥ͷ`70^ի(ZDH ^HHBv >ި ,n{h$] %Vp}F|ßmEd Z/׆S^cܟfTH'ef\Zd#=Uþ=SGT q׹kJw) 6s|*E-wH m+@*=Ǥ$s. No#wL;|6o~BYq;epJȾoSփDZf 8& ~SEǶ5b*",@@E~ _W7r;iV@\i m=3GܐgL3R_ 2kN cSeܴ zeBeC'}z ?\YkdicZ?Gͤmw=_>m`8B@_`JXyLgf&>S3[v};/:3Njv.ڳhZ lNO8kFǻC(}tu?Whrwa,p0C{oL_ao2:Kq |˼)>9)sX1ܧAP.Υ~L"u7v"Z.A0M)V?Ci4M/aOj-睬xiD s)H7N\sN/^"~+ kGZUai3=8C*P)gR?=G/I ]}{K˷CP޳)֠K6ԠBVwƴ6 0evc޿M[f|h_H?h8r )g?8iI> Pޫ[BϩNG HοbnksjOn\Ĵ|KY0s% ?xqwT5;B78psȺl7MoC+}Yz֖O~}@H N $G_ 0>m]<[MW  CA 2Г RKqD~ hKxjE: Tr2>ؓK GB h3F9>F#{'Z $9ܙ@8rZhHwIV(sstKP҄@U-IGq ׇ4qTקhe?}$uOMSci/LyS-:G>zl&s=Tr 5K!}`{H]ums ~nX'_d9K@>A퀽֡WIWIjIзҠT/ퟧvzH/d^dC}OcIl.w֙ ZT^4մO\h/[BBёl Bh{yc~\g_ųO u߾esedda@`z;&EݡxS׹O;~wXfm|@.B)@ߘ.t) )>l<%#7 v>=S::x@߂|k 1=eɡqT;oPvqAMd\;'QXUNٙ/Nj-밿cq5 gf|]N%kg=`E܃qp:yʙ[@~jp o[XZ -=, Xnaڜ_Ur8$ӽV3vPa6P%qե2$.6;d G|)?C|G(?gt{̯KX-8AQaqﱇч ?#~0a3g<7w:g[祥uခ!Pę0!b :ܰH@SbvĚl0+zWrTa IsELwXn\cB7)O:?8L! #]9ۺŧbns3oȏ4[ozR?Gj )iOo$K|z-R{x`péٸoirZ7!]fKZ*b Duid i 62 \#ާ Q>tՈ3g.dF Or/dM|,T PCT˸_Z# 2 |-k_Z1~+*FT "-Eq-N8- 89+_soɿso:ėʫ+tطka?<@㿹#dUȔF@>r>UAq!:ds9G˔5Tp9,n<0%[~6O腙Jc rrd8SV~}W^'`k }G'֌d7 e!YdWdYU*a1Ohϑ)K'`zS 俯eMA}4H /{Y''쁣2-f8x0$2߾"VyodF}0EE1>K{*ӁqB? yn_ǿz5dY\+DGQަݬ_(Xjԉrq⩥w'`}X̝nSseq?dps|ƕȺ WE9l񱷏oE9O* cĩvC\uVczߢ/C"(8~=7#2y߰y^*HooR1=8->6<T`$[_O[?U$Qp{+wc]7 Q_'>퓳ɛ:]b̝ 㿧[+h;Q>=S (&sN'޼H7]ǥ?+/qB9^ 8_mX И$dvmᇪoGrg:?Z. ǖ  Dyˊ-~of"SBTP9gҡ>Wk.2/1ذ6CKr51>89m/ ^{-Cҁ'U^%+Ec8m{sn ˥CBGGT0']z TKX>% FpZKBU KŗeX_p'rN5$_kپKy.Q/$UO-.*rE ϳ6OlQ:Pî̩[x~I"S:6Zr#M-{0d7UyxzbӢ(׮rQ3B9QteƷ Q̭[1\wr^fc\*~gi \=6qdmݬ(_Utʯ2mt~R@S7O qܯGDv~jqJkrи5?bz\uFu ӿعGЬ1-f>`Xm!C(?}͟qnWkO_4/enIsU?]p }ٔTbR6yf3˔E̻wGR#u)qe[Red=cqk i}r*P[2n3%'R(w*t:uf@w fH[aOn;yӗ ԔP{6\a3R/shͪ*L g5('9LGSί)!/yXUӌ4n87|6Ʋz"ȁ-3BhUCu]eȗ{O-W7YnE_Xֳٌ) tǻ+}+x¯[5AzR;`UqMn2k6Xyp{ø8tNB}Pe5.n-+LNhZ4 uI xn.wL}gG3+Qo]"x*378Ӂfq7]J1"eV9(?(.mU68 Tp{\sb4vxr($(WG8!&]Q}rT-$F]i2Šh +'PU%:R~o'fv/zYOU=(Y6q?{k$th;Ƚh{U>ԋOZW#P^*v\h$BZ,k< ΚszAj)>"[פF+v 7i/=y՘)(7e"~ W{Y֤z_y >f6M Aw:+ʺ}f}_]/-%up&WZ$\8\ *<ԉP%Q xي`{V+UtU3e' CBvSk%w9  ~Q/8PXОX:-_+zJ~)bkT8j5geuH_,e\͓&/ l5ܾu =&K°L3 }AVt>3)VX AAnuW a}irm52ϕŹ0gܒFgrD}󇨠eTi%=*dD}gT]).+xR nm^!Ijw9Fh?Fm1UIP_ OvlvYO;.yB|@d;qGN?N-@4n\u.$+-QZ4z<0oyI_ԧȣ++\72&QzvQk RdwSP'k]j1>mx B [8%hMBM;wPa"1q}^Glmlo}U/#=~V`.lhvż?RjBԟ.s aKy<ɥDCZGO<ܾbPxTxgH3zڐ)-U>3'$bƈq"O8zLKL?0nn,*d>>0T~LOK6ұofZD[;@fJ P5x&&Xsُ ˛g=Q>s񶬞lj%$ T/ k|M+1Iƀ\} Y`AE Q=OjWW~O\`n &WdDgwWcw͑SQoGj7 ;z^zQؑL-v0! 6a(^|ZH[VUԵHe'0Ծ1L o_\ϝGk[bL]¹mtҪ_a .x3 EϷ|31r cǛ'@ev9Gaa} ?l;zr-.Y\W'r}/ f4=!m L蠜 opWbIg;sơOİ:zS'7X,1uB,#А_5ޒ #E [XxQa;g=wYP*H(!.[|T6  %&͍pIM&ʧmIMۋ~7M %;`_$@"YwBdUAڰy'Ck}_%;(5PKDn~b}Rf+gv@;.B#I^;^)_ !ܤpAU[]l #ݱI }mxԛ68OyAu x?D0T4\+\Ptb)):9P57%7#[ O$_lyH: 0悦ُP:[L%Hr ~Z#KGI0}REI|YzL3?r spF[HH*M\HʯS3́yzݻBx9$Ŧ J?K6ҧ߻@h⨪Z栢{ެ%p}]\i Qe?"zhXd7mN5Saq|8TpouΓ&@~GQnt]{$aWi QN\y(cJk|5*|25@\4*9[nbxTcPba*OɄY;lБc}֏Kf gm4cHjQo\rYq]Ub|>f@ ŃŴqHs`|Pn26b٘=]ؾx A3z0uF?}fD.m'{ؘY:!M؄[6i2UZUcQ9$C~&lܲ=pD6Hi@i0=[>gl٨*PB,쿹} WU밾?'>cë$?3}-7~8~(6 ŦvCI<5^uO[E'0ߎmM,Bv6 +N#Eu3g{##a}; Z%N!'_=0^ QѤ@iY5ߝl dU0~:Q*Q,2?-n8]sR8zsH-Dn T '$ü  _lpBQ rc@@)ri8Xt(pʹF -,Q ׻Nxי,O2{jɂmτE۟η:Nj  C\XQU8)]k wvFw1P?U $2 OG} l]` U2'>ڝ6 ̙V]{;RL?:ɇ97{uՏ[BPd, q%\ogOI w_p%l^*N`UnLp_if8;M 'Ԓ2& ݨLdž@s8}~i xY_|x=3+,O'n3rSk-lzO>7+S?J*/zpiw$gfK]| JI;7"SNK*ہlʋ3g`}wlXK}4<3ev_"B RFpNv~in ܶ 㪰<ƍd{_()17:_e G+gFÓw j|ʒ˶wQ| [HВ'(0z=.yR("EO !(k[OKή!;%~ҟugEH?K3xpΨS2q~5*Lw'K|xk;U$E.O7ƲP ~%\r TEy1OWRu@~~f v>At{sv lGy !(_ZV ]J{Bcx K<۝4,53'[M3|:>]N}5"j:UȌn8<^8pOo' Lߔo;Q^窮&E ;>f*ڎ\sF7+Z\~JnI"ǚO]U}o7rҌnu΀,pTkKqUdcbUAEj5*zz8o,0LةMs`jT#D\j~pK^P[-Y۶Gl)U"9q`>rNͼ_~ ?"O}9;sUb-6 q'+S@44>p-XvkOpTm>:5[`p)[,=FƝ;6 WqY!ҐP,w/6 =\>o(%tm?wKԴID"#^ kz/p+LOZ|±;X>OWI+~L3헂eW!oMZ/ I~ڱISy2 a+d~_d !(91Ulmrcu!zzP{_H%lK_lsCN7j \/rkKb|'0i}۶1m;\ 3>$8TbNFBr rJZYFjOucET#iFWw'hm,M?)t}þ vKWt xy>soU:fP&/vt%ϸ>}{Wd{TtUk D\]* y;Ϟ&usc/OTVZ{-};.hJA b*fNs{emJ#qʥP 洎ū,׸Kk`Xg{KsVo跮 ܋hGtJ/IA'xl> OpɳH80 $ܔpdʣ3goNqd[$俟 *(H_Y (;(lLx['zpZD '*K>ͬNM{}F0ǧ7ƛK øf2%rm}q9ł/wK4({YOV9eIGDqԙ;mQ^d+&eoa=,+wfJKLzpVU}5QZ}ΨQ֤y2o}zHΔS~nM(c_,`/^v݊4Gg7R`j3wNگ L9 DZ c}ݛ閄,:YFƅ[ob|O[4;>]y$seLqV*6#4c:ce+wvJ-fn[B;֮M`04MHF*D1rfZ׭Y,cR?1ܧDk^0g\0 jtATxRx" \/R{ ߌ+쁻Yxg;#}sz g0q8qYg:?+~:uYgz?+~wƑ1ߢ}ГkBt5yKw UBEPI"ι`֫zkń6|uXN?N< Clt50wpTy; tlI1<(?NJ=V)!ovfTUL(\|PC6HcB[S C6d?+3[@pN/h.|j1T…te/.*xN>+e9tA~O\SAA_~ELgfϛHMuX~;f@ʆ# 2'&fAw y/b-&Z7w~5:#bbBƎ6Ùa-_&d1T~~ pFrۄ |!W^c.Cm @|C_!;vrqO"}Qណ?=^}20v0hǡutg[GHn_QFeâj9TH'Fdtȣ~xrU4 LbqQ 9U.>e-Hn_y8ڏH[DMwT_:Q2j ƻ:uwiij(^ nAh&ȴ XsV |GH,gzk~G9NnZ@P.yN(h%u UFvL ?>pdgݎ g3@8KU'yayܨ_CSlD2_Bt8Uc-Z\{}NȘ.[ʑW _}2wʓ{{oS$9\9ҷdϑC}o}JVL?B&ㅫfӈhrfuF&W#߽{Q\tZfhQY0cH>N@x#2eҧ.$EwXeSEQ"Cu`\{_N-+^]O#<+p ʫNW#uSVf˭JhŏE(V3eI(6yg^nּc۫I+TiOp ME4o_`^Nuw&; /;/Zvvp>exTۼ|zݏZim5Uvy Qx1ل,W{GwNgUhڴx Rr0'c7ټ@3툒>Aȏ Uұ.#1'q]ics23= cVw'|Hp6lhz</'4;]3464_#Cկi!Z!f̷|ajns9d 7mG!o;jmk~:zsvY\~ߓk(3Ê6ywγ `8j7o^+Kl2i.a A=h"q}΢ӓvǙO׃S#7533Owe6C4 9:М|-reEmO:׏N!wjAmU]?ϯ (_ l#|ujAZg̖qh_IhPLܥ^Oߦ|YhBꎨcZXL0~©:txd4\v^:/^h7G;DoshׅځO RxbZuЮC$؇uHWgyCF0yf/zo p>)j]O7_;=kwjb=glb\isl_jwZ$wp3xݳUO-x#l:d]~i ]XA*6WEd}_Z}?^ӖNҁb\Jd1Fg*^MR{B#Rys=F|?mՆmhKT4퇘w.^q/-?&ϼ}G<\ξ+@y'@;K*́]9ah_/\ ?+rC)ܿyo6Ĝ.?Y^¼tn|<(qXp8|Kg3'@EQhΔ Bt#2~mXKo%{:FrfMЗBcCmƦsR7,$'l_saK>WB}#]Zwu c|g\2ߔe¶C?J$b-Z Ϗ=a;#-oڌ7~)1[}6Ψ)iflݤLMWxL̠sxk5֘bk_&f s/I? W yB0Z/Z盻\K_ /wLy@e&)1/Mg_Kb.-3FnrZ>@ε&ӳ7!.ig%I0UPy^hn Kxk D@d^^:L)~WBC7rp6TTXs_ck187 o 0BCϏ€kj? t>$S yc7t8"e Ə71@wGe-R~Ȁwz7NJ6GyCm\P~Cwu. (FOo o5+0cMغh/n*@*qly :4<\Sj.C}f9)C}{+w(/} h]$>eml=4 }kKA VK%<^W!kUof=~8d-v\- ,c˦WH;!<_ ˷"2ҁR?OZ߾Y|c.:@I(\rUj}gt佾/S.? {J@>XFw=ʧZ`lvAg}},#R-Cu I/z:!]Э{kCN.9oL=z.+otۧ ٍ {;u/gB,)9ۜduĶ}_K t!ݲͬoN O=a,85"`k퐦:Ia3e 'C[nۚOP_ꇜs}c>+K+Vfc%/6~͆m䣸brl;qʘpe:˵9q<*je^ɱ}(xםTtt-MG#?QnN7Cj~бh H)t U JX2/Ylh >^s5yzV7⻗mliG?XcZ8llmTmlKܲ6R~JBէ{?eƐ`؞Ss*(8^,/`GB/ x!~t7_??JOl.iSNb߿0ɍO!@y4-X_ϾՇсv?YآW^ӫ d{ 䎯_12MG}r8eV8f #9q {6jH#זC{S79$LjT4u*3A=/4dQl0]Pm"#>38^Dp`u!Ȍf(.z0.vO?<}) oHL4s~nCehx Ya`W]R QsBrwS&o~;,_c<[S}plNִD}>&S~"J10?K.p;ܜ?(bne'%g>2Ajj,oFʁT/!}0 _gd6`̀Uڻ>v/.nb/CI_dlAzξdɀ#-Phzy9BE[M<5Rx\x}1x ((NwiZ&r: xq9@waowha{aYf5P}5#(rA q2}bg׺+܀`#oz~WD }N+0œ:\2qX/!\24E9Z+@n Ola:mo&h}בy R7у(q5+^_ }}HmOȽͲчr};(?*R E%> ^Mf\pn$M@O؞ly+*Ѿ4ZHK7y:f_^jM<}&OOd΅JK6>RrWO޵a(qbC᛿k0}ӈa1[k^{ipX~$d} v˥f/x{U<};pj>YIL;;}(QM>Տ{g?8埡P|ӗ[L4? o[0u{m^߬q*ؿU S{Fׄ16q<{Ȣ"; }]KoN!jI}~L/LZAߴ2? aCIRVU%~wU]׆/AnyV`#&(T7A>Zw}&cu|W'xLGU%a嶁V~?xb uO1U"';~|)h-~=\s'[2:>YRnz|vMyfX_+4MA|ه}mz;}Fm@} &s4S+,_;4p?T^0gO/o]K /6!ʿ쵕~Hha15 п[?Ao%Ӹo i'^'sM@z,>Cm7Yw? [ZGfB(΋qk}v*9YکڐMq}b)4W}Fuxnەk@Ğ(oxt`^ozbdU]r]7v |ND5h+ʛ >͑y&WWQ_9run/Pg~5|w!YqYa[B3j%M`+E>eϕq_h{`&xwJ7E7N2D=œc|6i`K!wtӖ ~J;߅̝p.O ˾I׀&KUCEJ+upvQ,?,d-0Qqp]u%m~ s֫*X.jl}Ʀ79KB'kISI>aJ:N%x$:)h Ëy ĝO4yۍ󣮎RX0Ovs3xm3:%מ,,򦞿*W@̉KrTG0[ Ưh\//uw;XXp:& |3D}uS!J~_Ql_uLPjޔUж|0Xxה@h l |_gN=m6!f?VOmBɒKeJ:c\V7]T`gR,C#O?3mBƋ]pCڄ~} ;I^>"[wCN^>SP<ʹ_&aC^5GMB|Xc,v$ 2YjdC*)=ۧN "O 9o93»g=OϹAg Pq~w/-翼-g<]&_Y-߇^OxyTyFG.q}g#ù[q>﷍ￍ>6a7n~9aMާXؾ*MG}6&[K(._ C}Xn ^=:rγ1Ui}B}!BrMTCP߷/F =W~S^m >[gH( ;jߑ˴¦_U4BQXu[Dh嵴P]yǐP%c傰paCI1u]1}!+%_xe啡}ܵ;7 ?ڄ΢Od UEUxGMhOcGN瑿Q".aVZI:~alY />\^~cV_1̦W?@52 R7T.\S86^Qa 1o\r^cD^cmwmMXi7(PSn_lG_pYUa#ڦiS6͈nOynW~#V&qIUf깟j7 A.f( YɵɱSBgv!o:O + gycwC o+Q " w3wְ5 ۠32Wltsr8s(+Kc$wBW* \ s޷OG> :rJ q0M5JUP~ғ'C:<>糚n,nB,7&&;l޵_=YEӫ7&Aݫӿ\|<˳|j8jXcȅ%,N$Cb}wk1)_t'N]R X^yo/[s}\m!=IA GO_"/H7HE;ob*Zkz.-1UR'Ѫ nì#XkEb$zn䗨ϸ! qXuQ3ywPDFǐ2Nϧ?6|S(Wi0{fde{qb{OP6 {#@M-hNm U "q61WnPtK-_AkA|^'_X6CUd ~{wf~O/(g)C|߽|#>Yb,dڻDA0D~>%hGb6w!lyGbۜ+O EOjPoͦsċ=oȟ?hKv}8x~2cGC{c  C:HW+)Q;TW 6qaGЗk0¹/ ؿ>s0OlD[p] B^'M-[ i@;}w3 v-()% ^賸fd rj=QhkdNP cZk`oj)&4;B-k(wc{ъkt2;:ςƿ^2^o] ('EȂ73ap%P|NmQfj P|% )[sK7sbLٵ cch}?8?8ST Sݑ>؊9caʄW?9|w?9|ϗ?:|Ϸ?:|){wƏE/cDF>Z(Eߴq?WŃfGZ/;"A;rylӦߜÎ@K?O ̻e+=Mut]}%%ZaCW{!Ȏ$/qJJncWoā5-vʆIr@)f-v XsHg|]wn|}Zeދ^oE-]/J<#e&C|ͬĎ'>E(*f9B9q3b;I,&!FtI󧈝4x}lG\ϥp v-^ V+O4L}[PW##NtCM6e,+,u횫C>%,YpY֐h}_g>ϭEnMփl8<}~a}1ʗd/6[$L3nk4z*HsO;ͬR|[~#.t(qO6 )WL!{3x7IGVIW`J.eH<.Z_&U1CGͨ)o]U{?qKxzil"CgpsY'/~tC'{tMRp Jd&~Sk {aRyNB+Teo 0,O!&6q'6}T_zwsGhSt#Yv/k?yY[$Ϭr7r UeE{`\.[unRp<4e}۷ mn@rx#c76ƽGQ `!R6h9@s4vfU #tI~y?&^b =t9@d`N><7t9\q_+}^:F졤*U!c|}IL\^(܉jڠ% {aC˨ov;}TCrxv+ 靔\lj%,w- 9l.v9tGlP.D9W e)q,tڑ IPZcW{;B ;įbgPܐ^#P}m]8䝀΄ Vu8u0.]?^u61"҅ۜHco٫p_P^]9L}[{`}^]ӭ(;@U7m=M>;6=2:,]{Xoooٍnلdb,[kQ蝴0B~vVn6sKx>g _MxGmҩ~mѱ&*\ŲS^7>ĚeffU>$3}gp"՛ɶ^R[H\M#}H߆o͠D5ta,K7Pu_>Y#mN3H_z% GEwGm`_JҲF38}ŕGaw[_tah؝.I::>.4ݏa3y4{u` ȥ=3@'>bRA_kՀ:)A OU_g 00o7Q!Y @eh9{l Em@,J㴽-H} рqGW".d={C{7( ñ>#@xjXUr5T\ng p?h6P$`?3TGN@ymM`di=`>K^owdA]96D^!+ vcP[YBs~\^ M|PrMV)bDpc+ŧWꌁ=/Vh}EF}.~Lo1å߷b?=|l(/cw%hc}lahTͤכḨgk \jN3IxO1hŃ"IK{5pi&ܟwjk?q^qBa{NRL!f'no~okSw~j`ڔaw`Ye U[y5!h˯W JkQߘj:K¼vH_H*/lNمf{:LL gy^Ue^Nx)Cj\cj^shX(٫nsvn˓f*dQ̈́z6tꋢ?s<2Dڟ0D X2Sn$J+UX➽?!/un'`oW7LrA%y!ϓKhIU(€Ս;Wb|fAO-:Z5>k/a9$_&TŴCq@ҳd x-vޅؤlÛSHQd1Eߦ؊>> laKT/}ptE-!īg۟' "L]((%6BD6/} IJ͗ Yی{=#{;bޭgdH921?XkZ#OŸFFH:MxOy } U[e9XJRJ:d/aVe+VգOQST;y} =4'ZIq[/Z#!!GNWqs01U%F]N ~Z_ۦJ2Ug%a< <1G Qլv7cmG"NO}5eyn%X5#ztz}kk D~ΆhI ɪ _=^Jrs_,܇eD"?pO6WZf#RThjXT\+;o3ַȏEH*xhDe_ᛛ1]}ƗPgL|m@U+*nNKH`^JKA׏~!{NHK>q\xnhG S~mBu<aiakQ,$ZP5v4I%뙹D\^"_T猯HW}m2`l7jv^Q1E{/ʓ+ǝ:a߬V0O )ora(45g&e Hc(?Y!Jsr#?j LZRʥ 0GR6cN#C/F#6, CU&'D +3|f|:#Sdyg"&c3&O~xqP]c@1[mXqb&Q8bԛ NƔ4ɻxP,gcc&7s!~и8/dmp҃'vDw!^}MP㞔;“}\$_TTt~ގ؂0$Ryu\`H-U³e#&&}U%xњ[3>?r~^<ɳW #4>krpt͋ ӲhJe -swAX}9krzJabB_Jgk1a\e2a.tr`1aWܼ` ǥ꩹pܮAcRń6&o-e/&0M͈x YFN /D T\PkN%S+B.&,qLhDzK [H\PFH#x-W(5TZ,#c^`a%K$g$Ϛ WzjG6 wFJ%$7g*P^+)i$=UF充>/j^5hz$"``ւN9slWh5Y5`B,ہAȪo- Nnf35\ / cBP[ZoH2^@5{ |qcw ^j0A1maQ=o'vwoz#zYaUWW#HofL}iCNus簧2 7~_b@…sFP`{hy-#r)mVw=@y5wc닡u$ψl[Xu*D$)rMA~(0BE $.#T/ZYB[_ YĐ1-a'FWY!qc2qqᝆ>widRSLt=7b3|>z 噂 OǔqiN.y1o10x}o$g[>U`E߅9s57/߯ÚC߽mXҰd`ُ%K>o-ʈ/Nm7UȢT[~iלԖ0SdC(֮nTN|ȹ\&էO#HhyJ|s&c(>70%L~j'~{]MX0Gx<߽čm܈o k!Aď8 JAagIBUx-E2oj!M"Hlj=t!i&i\vA{QEO ߋv_ȓ$U7S:+A}ģ!N d@O*H~P` 2DzI~Xw]?^hU8-Fj3[=i|wu'ϻp] /wqr'@Xw<8) [O}ClPOkio=N~_%tfO4 "bRtʢN%dTaS:7:j#}i`} | M>{luJo o>Sp]~KHSW[*O|þ_Sz6Aަ{dWD\7-[ֺEj5cWawPKҒh~twuO} F3Z_Kk=o>iE :DZ`uʿV5R%y!ᨚZ Җ/>x8Ols|(xya}:E%܄U"w'@%T[AZo'Z~U&§'^sΕPeƇG1eXL aS\hC*Xp}(^z*ʾHj7)GcP3F._58Hb5ꤵ{;.@YΝ{Y5IIIXijJij$Q*D~rxMȾ*"=/|)l*$\v25 T/aU*;gzukv2ױ_cZZ*iI3Z'x9Af+SɅq'{wG_{Jdcc}2d/.nioyUUݵ)^*n|̺+bTi~oT!LֳG(oqG^x~d76n]Z#K pD:y_x$<An[y8Wh{X~*ݴsH| ?rRpy*).YܘF,LhZ4H ".8x MӲ؞|b{C Ʀ_pԸ]Jw|{5R$: f҄*sd3dˆ됮WϢW{k-'ډ3:E7&x#ޠBNFc|~W[z<9SKGtݐ_wA6ڜt1!~"c)7ic6N3S%,͐u1\}prEICwz]y\Vϛ4 |84m:[}̜'f<\[_߷9ߡ5% ,QXbl-PAXbyK7X|2mc"E*-,XL8bqB}Xq!Thta a$ CS!6v0c?·}&mx9-n]Q0Eog-}?۷we='J.{xTO;qVySS`UJO^ىSNm XMBzuåUpMbFӺ ޶R6 "*0~|JڦʿU!s^XVTTy fO: ڻ:F6a^:iMi.Y'/y p㋾~*DY*"(ir[5/n1@h7,6A9)mp֗ʺO8Ƕ0cy=^ВC@MZYˍl伉o9Zuq=k4{#[ LO,aKc rW"`GڵtwuTۃ:g5?S])vX faENߜUBYn NWr*XD\U1_3XEK#}tH_Tq6?;;T}}FԞ-t+G@IW~cm~~a+@ؿ9לb1{}tZ ѭ{T&>e]VՀǗ|G}uj@ĉcTaʙxj?yeÕ٫T|̍U50 ؞C6֪=ܻS0ߠnK? ~5j=9a&u׶wZw `a`HlI G~@vo -~ߘO~.0&}-{M/xN&7w=PߥLtLlWet+ߎd̷ JGhF:\I|3BgPԋ)%AajË9p9K?ED}U>iB\F(泆gS,#?lpggQWm *[oc CaW|;k{ncΑi ,[ʢȽUg_^Wr1;.ETvҠT)iXjR-I*e8* ?6fU@B(3}V{sHsZ 9.8ǽoIՅ 7㯘%Lrz W #)l?%M77ަy -sZ"Rm=p>a7rۜхWNHTNx\Ѕ~Ky!_`"}E֧p~W*9悅8?Sz>aԆéo"^!MdBĴ){/[*iCj۱?^mӅp. _ᶺc(X_ Z 1^ lzcM}K^YTӾfz9u#U׺ >D>@}߉}5o6BH߸(H<}?3;s[{]nzO@sq `Re%7 Oӗ0 ]TYjJ`s[ kܐN_sh@1'q!:P;#_ꍀI{f:l/og_kB󤇥ݸM E7bpa+5֧,_+ %]ȯSJGB$_n=-T!Xc;f+1f/: BA}i*}eLrKo{>&+R& &*}/& 杼*ЭL}fqR`07t0qDɻy&o⥯?3Gq9q QMQg7~%/ OmMg,ա^~_]x%<5SZl{VAqUfB:oYY)kMZ_t6"N<\{✐ζݢH˸8|K]:DgF?G~rZe"]( 9oXM?55HfFCjDz}@{'E+q|[5C _di\?aG5jGK[$} nH>{ z?F `I̪+`>YާiVS}ۮk1{nӄv^_oc{"7]yS k{y57)׼៽hfL bpH¼ q">!o!z_Pח.ct,Y8}yp<k}TZYf2'_ϙIӂK ? yO=D]>ze'Ǘ8"mz\ G9Pr4o#菹c{ !}޳Q O"׋~Oyȟo6&LCrAiZD7rE{5r/ ٍK>~NS?Nm=DݠUV<"pEMy6QίTc:sʕ݈'d;-!D߭t|uiyF _#6!(j/est k_/g=:+ݞJlӝ/8߾sni_eDp /͒V 3e\m"+[e<GR9/ =Fl'KmvŽ[8tU=`Ow3|k_t4#p-(^=VpWΰ)d`}"-vۃuۘ>s}ݻcmܤ$ 7-M]o]}f٠x3[ag<#;oޠd[Eya$O32-PB OpD״>~t,d.θ~6RwFq?XձTw&bΫʘͽb},%Np}rG (UUf#չ<[K Jm4g{O3XJ AYS?T+!>l$߼KgLB}H@?N\O0Ko 堳/_w EU .{+pAW؟Kׄ |LR -A:6L3WEvm}g7N{0מ7FW/4MA hUL/A@t{Չ#쾸@zk$܍jet"7cTGs[Q?@[m '0 3Zjsq=ٶ%+K7s ?,p'NǼ}%!6|30>:;?Z^g|c?9$Mc]|srnb&rUg5\B|z%<{ xr_.BKpf;l<1Ǯ%f}NJ {nOJB }+m!z# 6!?"1#bEHbR*>}ivmӪPDE*Q(zJDQ%kRzܳs=jt7yf%.0ǚJBt%`ave6&lΔ l 7I}ێb6$~9v8I{#sHzxxZ6d:U4tAz-ۊP6PD++G15 D_t+3 eR3|G+S*! ӟ ~ahB)S~j^gId_ % ;h8/Ō5lߐֹUFgSwIl[_ ʵ64(y8 O.T}C DV{ҠpN@8;&Ԫ#bN)4xMÂ!uMV|CS vеwiA}T)45IJb$~m  d-z+Gi]xiѷ)܁8z3f gTM搯w9DCGZm19 ,p|pK7lx[o>^[G",*z!B,d Q}3jQX l+m X30̐8KH4uSKP RcG5]D}q^_ޣ 5 ` $fӞgP^q:ƣ7!>py8~`Ho%[k= \O/3}\[} PmM 9V<Qmz+Vs:@u=9V ާY䒯+t@jH6@ГA)̟ t|x߽9;>; ǯ;SǓﮆݑ{?$}POo t~ñٛZ2띁M}zO ܡYNց2[hקWGJ}_o17e qv]=z\SWp&}m5GܦY93w۸P%ۺ-7Q=O| B='ˮrXE+0О0R,+h<''Q*'[U^׏0[ˬ BB+Xb ⠲3h߱*4"*TP{l쿲m=k4BTcG7F%`T if}Q+UW*7C[{6߱"z?>|eE 2a^.m< *&1}=fK:jAX5GM޾J{'l# XL)]n TO9}5Ġ6{SrM&󱆾]5Y#Q^w"wHuZ W[CY3+?Yo Ys'?Xrlm@xTm)2ޱbrkWk~ԝ&A˦_bMpכž3ɜ<>&5:{7Y۰DErBrGLzR7k6;Y()jp{ (;9V& DK O&|R"dq}N">M$=qrw3 'i.O,5ҭV"7s%s9WevG|lj/LR&AO,T%ʆ.i?K -΢[ֆ)]q|;þmDG齓sf/UAА={ eɇ$hvʙpajof42,J\pTv Eg \3eH eʚ\lX\j=aW15B1eav[(STU#?Lըyg >m@9ehZM<83AtT9J>W#83,lCFZsE"1i()CE_zGWqa!wtJ]jD N-_FenG8;/#1t!ſJLk'2R\Dvz#_zBdS\ׯEP%kEPj褋QhVTͳ5 k$y(ȓyT8u-%uJZ]u=7mBb@ί{TBњْ#/u)Q-X}׷?2ؔ(>oc\_փOR!&Ո)BwM5LuS<-tO]X.6jY& ~sx|P<ұ7?/Rv R5cOِqs|lEmPщS& C̜.g۾焧*PX*'+UA8Nm:"fGA{(?dacX}R]*PxX":S ʫC$ƭcZcbbF9xqkG>zc*=z2|t~C Uh^@n0_?w^eht(q9}m2}Hmū%TrwPrmk=x'(p\wm,*ž((Uľ.HPa=S-uf7. 2oT۔ͅZy7o{_x?[^xx_J-O<݉8~$0\ D[_DvoBUl݈kJucyo=/9(Rogԛ'_yֿ(Fl^6N} &A)֨j@w i#.`%u chڿj`@#o|bL-]$"χ LĹ?qV"ݞ;$ CSeٝ雼Drc?7K1FV4mM Wճ*/2ڽOU.JRlL'a1^.2 ~Cs ` ^)@I̅xxsn˟X%֌sC~"9!ڵ- p!/e$V$?̉|4={[ͱؑ#w؅ 7VEI]̉͗,67|~Q.1)!*.u&}qiSߪʣ J/HE;;#?]MO4 ?4qNp=d@[u] y}4-LhOzde=hqݔuhdCQH)^}' 4^]"? !>0 |P1JLb@=g̮b  V w~x?oZޖBbTW=UZodk}aMU]3KNr9A=I> 搭{YK^kc탪'`ٗ~^6O36;Ycke Xp$p ~ηR<$d\KҲ?9PxŻ N,l H[ڀ*[dlmә5 ;Vh5˴_,uAL9Xn_%Dyj_'Z&k*@s/nXTBq#U' [ގ@Rj[i}ז;Bѫ#%R5;X[9PuS(Z,Lً^!n6x030Bodjs]]HF_ͯ 1,Oh/hi3RSqu 7~;@Gw!])r韵7aB/>1tX|ڗ`kl\^=u@$IeTھw߷,68?j|ci?1zJ¾ŅiG ǙwVCzUa~%\m2AME,#[AM"ϸ8Bڰbh| O3Aq?sjo%nsY(O,?UnMW9^m ; >O LP)V6ϓ4 (ag]%zaoL8]f|*tL;X v8L]VĢ%=Pݩ6{:pr~.Nu WI~If?x %|Hy n[Vّ~]2 (3w2~;!εfvYtu "='kit^dVjc~:K_U/3}ϽX{ڰ=o Ъ(: g߹wKw[0>.mm:yQoXf'tJrƐOgŽyAͯhےN;.R){_Ηp/\<gW>߀kggY43W y3\MiD}ac[MoAQ?hŏ}nRWSýsʤJ#j4QŽK6tǗ"_TVSVOG>;Ft bAf [Rڧ8u=sEcdԿgl9PkWݷbi#Pd;cP_=|!P'$zck>…Ox4#u[<n4[|='zq!}e3.nPU=sEv AD:Cm|2^xαZπ.k_|gϴrsTHZTށjW =ڵkeeѓҰ&Wf}qY|XQ+Rse>oY& Hq52N?0e$Ѿg 7E۞@GjߵZl;9)/轚M91l9BWs1~"LV@ߖ>T[@A&c h%Zռ qB=FzH۷~J1XLN !]BCj@t$q1i" ," $M$m$=$  )iRR&.HHWꑚn#CDFzԋI'8s" ,"i.$*waH{Yn{|-tiGH` I{ow$/th_BK>gAYOpRDW?ǍgC"5M0bȿҕeHAa17s^.Tʸ색uOCfz숏Wڶz/  6~o-=OMشp9yۃ/`uLd/ Gwrl,]ors,iρg%*,XH}3II34AŮК(Szn?!Pb1z pȳZ. yc|9QIާ怸Ԍc{ƸP9g`5]h,uPo=xm G/ K7Q*?fP_ӹṕvh^V&ِ}> # 57f cVq,*ޭ.U.ר8!>n޽rRZzߋ@Y% J/s? J7}2Aӽoô˃~.~=s=]ARkO jZi3>)w ƴͮ2魬 H ~w[2rǗF1=$>f`6H)єK\V]ի8D0ʚ ߐyk,9֌yhMaa~1P^A5+m[ 0 MW]9py,l9ͻY>N kO, e\`Xs`z]ˆ^?"́}{sAX3M3pA›6hnpB1={NpT ؟vIuCٔ=(/Xؐj<+֜ +|̆\N9?w0)r U-R 5ğYƽTy̆ _S_zoHbo v'B@9=֫=bW@L`Y%N@ϸf`y{A:O!ʡ-^j2I1iB_o8 %wapEN( ؗ[C|(jawq}"x2 PΖ [W l!:+NĪو̗AUzVx8bp9kdp`frH2$5]dp(OaCCF9MIlhUyzH=A.q~L~myj$# H6XXxELB=0t|0InE(?W{KMVkh/*MqB#@n/Ćb^'"/d SSgKFw]FO?}·;>Ӭ0#G1̛A1,,/=lVUL\mǏa}kaR׫1LJc{>6<e:#=cXɥ0檸kcpVJu?8PIk{.T 8s\ X0-"?a3mY>"޲9L>TjÂ+K2<#FIMLX 麿X}7̳19gQ̂ʂfhzb2 ?8Gigv"HЪPԥvfƱO3덹`0yc8\|a=.wYDkOHH8{;>!/ە f9^zCO:jmSžv[f4څ\`hx)xgد[\M#79f-Ekԍ8p 7,G镉+InxϜU8eK5Axnps|\+ XWV3zd}X4_0+t9=;¡1KTYcM8e Ox ˉIr /*~r+"^=vK̰).KpI8g/Df6 J>J1q;_Bb.`HhOɔc6dkRїLkP#2!}`b8v쯉R>`}2{Sf1Ay/Ww:}n;mn%s( ;gPfbv# ژWc68CU7@0PUxf^|n z2MIbL]X[]dt/0Hld %|)amԅ;NQ O>J_( Xҭ]@ح=a@ʙ[Z6b0"WRj5qx(&=4v̚á骋 7wY%nz yvZ 9ywOI<'OJ5P׷cߑ93((a%c5GLEG V%wTE2;Zn}]w$ ʪNmw=+\Ex/k[7/kϴ}!_Οz3{mrOهn }7+ 1{'gjҗ;Au]CG0_" -;;ǧJְK\h Dhf\ n1/o 5ZWxs(-U[ ʛn&[ WFW͇]v, ]vO+ ѯ]yUcW v7=+饗K]!d}ZP(nkr6@=9׻0R9nQ( \2+D5[3-b֊叏/˔dB@j}C.=ːČ  Vq<^I^.zDž۞ L{PS wq bBNԹ`ǖ^%?3L뇀`8o7 Bgkv}xﶉ^z@(Fkv)2!dJ.O"џ*L(-5o,G}dwGu0xqPQl(^g)S#&jϾ[C>jj'Dq2!(?7'蓃4쌕_Aj!9j?u8N QmAr|OcfMw/FLo㗪l4PO.Hv$M«JtI|oJHVp3 7[E7Ec2_ BzȅNhGY8p[#hu *jXfB@{qg;*x>x43-A!u#@>s󬫬#,/+CNF]ؿ =NHth1[)*Z eϪrRڭg GSj Y{Qz3о1ur?6J7dVsMd?Whq~לPȀLքZ_<0l3kNKUfa]~te+9eB"e _aha_#u >0SǧI qP8Lӻ]-?Zφ3@zBH7s=akzFo`=O"1@,xd7M5oExLuU!ɓeeCO:V +.o)_ ]Ƅ˒on(CH-&wZ>q^p`8 ݶF?.7_~6^'dU03Rl/߽1 /w/* E~'@tM>Q~k w(2\ى}#-V=a"}] '!xhxMgv}/-gh{)97l3[NR?R z9|yp|jaS3D]{\};-RnYmoCr_q#.8wba"ܿV>:$4݀"}<_zUT7 M\lG ¢bܯ1kf>ZhTN4~ Hf[T{n {1>0!ÔՑ€Y'.;0oDL;_L0Qfb]/c^i9mбb3pkova=NT'QUZFk1;WmG({?qz"=קqZ kχJсZꑐudg#{ܿVc~GapPƣih^~J wC˙/8AY))ҍ2Nk`r$U\aWz:57<ɞDJt<'|v_Ev8B~j[UĶ;`I<"9uIKKo|Z 9y ֠t YˎLj~"tn]6Wϑ:ŋpfZh/q"5O(e2tH%|gHu".tKߜ?\`< sCfCSXM|'HWr~\=/N1yCB/}E#b)8`I>\%B_8V_[ x]/Ď:eǵZ8Gy~Hv~ZzzYgݍu"DR!Sn9@5uH~ nZ3_ʹ ;>e8{=Բ$RDLsw*(T[byK@<]C2v9e[ۮ#=.5.vi\7{-g؁~Oi'Ba~=u~T;t%Cuv :z. XQͤ}n-u Kl~xtd+1UAbަ7q$ܳdwo+}vk~^; SwCn+;`i|7`|0ROCu 1n:Ɨ?tA:NnlfRtSop퉷oaj_Z&$My5uv|goxB{b]؛҅HײGo@yU*srgo[В"YT-pOٝؗW0ݛ骏{劃mAG=eQ1f@]vz.Md+mԲ;BL g@a|ɮrsdݞh@[sE=;(gr_w liO[;')qE1-KĤjGqu784檮 EorY0B 'ޛnZZE{?zq8왉7}̟kھY@_q:bSM@9(I2g_%g컄 ܗY&VЧX5t+?WtX8/{ )gbXSgܗ\j \E'Lqak ]@)[M"I /cgn9%sc](9m+N {ӻ71eG\P>ֹ}S~s ^!vME;vJshS1̥K͡+ 7B+@ i7́NcowmTq>m[+]Sss yje՚I8^F)1pQf +?ujG,!q0Ӱ6l/|jJӌ eWUr_W[BSj n |I*Cn 8ws"Χ,nNjmQdb/7~67K}<8ʯ}y;z%D:m\*a<O+>(|p6Pf+w]=H}$݄5mDL-:yBoK2mۀI Ƿ벀kE%PK=ZYeլ`|0#JrV@ޱ7#-0ʛ8`XksQ%WR\AW{-0ТߎЃCk;v+,)l.=*4#eT`6 >c{yVPzqB>&UBcFVP&uf;eS_ Rۦ06mx|~|CU[66whb^xo(xd 9::NllɆnՕ*7}ad{ | xz*Fjm˞ X@RB9p|Rn L/tD Nl`D<|VeM%peC BS$5K1~=dS-{BmH|&~eamf7oq~Y%r3HoUG?D];p St5tk~kwoExGBkykG|^ﮂ󅎐i_*&%\}ߴK9Bgjl1Mh~;€W<.i0t!'S#ϱiPc% J ;KfJSU5>gN#6ȁ _gOwrX4?hӡknz{\R{0 )Jקrƛ Si@W=p#I Rgýͯ:L@Xl2şC<TYj%`&> ʫE堜/;w Y\^Sv9Opͬd9(r^C, rSqк_ףq> Je=S}U*ē\=7lȌٜv1^Ͼِv?r@\_;7&p@Ŗ6Ӽ+޻+D Ɔ++8sfZm҅Wn""tv/9T)ԃ7&+?[*b:=kEV2bճ 9n}8PafPn'[s@{sO`,*; R[zN|aZ'J&_)ye',_?8]t'TiIxK3Lyq*A^9"€1c=ksǍ OM`߿T= $ 1oavyq؏ω,| huSO}.Ds3圔|@;twg?:: .ߡ-Au|Ñ?8`hL~;=)Z`ڗx=1:o$=@lCS/2$y-I 0I}_+bgˤr}|eM<  :|A| o+U0w3 Z-= ^>H_Gb7;<@#ո$E.vp9.s-?ȟ_3gO,H] MwTT,^ 8zS⶚0@{4}I Ee/nd#-/l( YKv>!2TTT*0(E(%DT*%{1:99Y&<Ѿc(N@^f 7o3^ɇyx8a\uwV97-TnoGMGz{~rs)踕ŵJ>uYmB|9zZ Lڳy}@~/-fڇD'8-fRoUxɪ|o=*H}xvGȟ~?^_cu/ KJ8&9P<&KuK)`>rv8] B}B’O8T_k!Tx [՗py;0.>&?>Ƿ:ޝj>{<{v)ksZ#DWt/ҵ`Zը'>SP"6OsY~)%x' +:,;ruG%o^sR mFʇ>P_3?8w]dIAKQ}\p 4zN-FK3Fw=6JO}4.-G'Cnhg<}]Zr./&a??E}9Hw.~-vS6ybh{4ґٺ!ordr~|f}+'>qzF7|LNF;2ț*8m Oe=#l~SQ c'UST $iڸzj*q}r><l y{t.ؐ4Y/e ʠMo5D5FJ|ᴈS{-DXa )SHhH_^t,vRmnt_|9{ߙt>{a?3oS:J߯_)Hv4qfO?a?JFM-pzO9}ժCbqQl~gAcXE%\{(z? Qz@Q}g죙[{@Dgˠ,~MN[oxZ'=Z^b?aA݅tcbcHط0 #ŎBÓFf"6;{@V؟X< oa?z*Cy}ӎ;k.h*;5Osk3`, g|w~'wʥK`?CT8$dIzCaFZPDY|wLt\-3oə-c-;[n=|E{$&io ꀡ+d]>UoY\ A[`_>7b?A5)8ޯYW at]q!c=I>cQp0+Hq#غAPx3(F| zqR 8 J7~xɧؙJ%7_gt:$?(Q]߶KӰ?UUgԸ͚ԅGv8Ah0&(]m zQވ}/6i`Q5[~%EEQXfhaȍ3*[ =_}r[oǵX:CDe7 M1$`"// +~ybnɪD7ʌvB}cYS8+RFx |mbw5 ue3 h o2w !]ڞx&L۵tw@O3IlvI )@\qxHA6:CDI]S/d3 =~1þo{w;4T.vAm[P\31I$Y[a5wɀu)+:j%Egz#4QG+Mo@;|9^+`:@?|j%ggwsgk񣚎xc'kn\|?;g閝pn0zy(|SYQVINb}\5k=Ze9=ou0OD )4:)0<YFŎncG/G%aMX6D;sWZg{.,|nOG/uk T<ﭝu`Չfp7Y\g1Nc▿T{.r}+G@PJ#g^˱يqN6<8d~JHـvX ;v[V_iVk 6_jAk sBj4m9#u̫=XhnU 9sq_5X]ߩsl4qbBBt3= j rp%9יDB}[okXA>%yt+ӶZ7j#~6bl;/lO4̳js?zʩGl oiG⬅mxKu/{-}`?Voyq ok77GŽpne]=b@]f c0w^FAڔR?|c7s$;b\^*] [^Sئy8 ;eGzέ2`Aw 8SA^2l{h6{ tc˼L>*ݽx,~ kŇжG=m}J+@?p;#o3/AѦ'pqC-Zv0k Bg6Dہ⒫ivУq=_;yvy!)iգoam?k RĜͧ' =ʋ,gk&=2jz~CInJ&BʄژCK.*{(nb)?7LW{ MH_vPeuPN :Ø@6K4_Owşp&O. 6RB' '96HνoΤ~ߠܟϳ70$] SH2d ʋWQ:a ywޔ8SwUy71lY ԑWd@诜K;N@ٱ]ĚJ=ӿ9x;KrVˑ:Cζ<60+/zf@ׄ~]=MHc^?ZCZNUO+mn޵3LlH/*X * _=ͫrsZS0rb޸n \݃14iyy*iyk )?]9b=Tq|e jyɒXy;]ˌG8JyQ9h_ ùZno ׇVqf/- | iLLY\A nWM䖷0O}|u3GEގ%kôIL亜` z]ң񮷄A{yțޞ7gHK.sqB{(:Vy 6C~@%4[mW%PQg?:.izϮ!7\S@tq&9{ۯ.C 1>_Nf qmK Pݯbs/_E/ct^h\Wq%넶#sp^ P؟*Sn穴)'ᑘDI 1H$t]Տ /,fGR]O1y_:ORLn94b}nai,:oB}"̜lgYZ.ExwEE,@o{,aNV'|ȴ^=XԒ6(:bVx- hޥ5̇-;}$*ܒwZ@\g?[T;Kg(ޚl(sʽ-'5p1b])00~I(旷J;^9D0iM'C|_<,CFЖ:wצxˬ*33>KfG:lA,T LSD_X**0s7Jy3`^,~ Lͻ Aoܜyi2'֝^SZG㬁>&E BL"btj&\;=Gl{兓f=W $p*!7mw&ڶxF{Wx1IJ?4Ǎwf0w~ n^>G ㌎ްY_IoQRmbCP޿~,;8EuLv|} :J BO]i5#Ig.|=:9Gugpy 嫈OW/5^O"xj ۨ–l|#-,S9y`|}.JP<d[ ABݴhC@Lĩ;.*pX7mtA-e ;l pʂ!}~T9Jb{=oMw$J8>* 1RxU1*cCJdN^s}-b_ˁ.kO4qsϦs,Bd*ytf8Ģ: ?#}.mw B!';@N34 X`2'-32IA%)ę qS~Uf/6x{t^ܽA!}tqm38AC~CiNp'0Uf_Wo :TS8VPMK|_!1}f5 "GS=F(e% fxIhXG%Y'[MJѶDB\6]Ky!fQ)}F͉[(Yh$Ҝ!Om('_wa+#=OC,d?KW?_2\F/uƵu̾!Hņ!qA!_[.▉0QM:ԋط[CxZWH a7w^Cbn O&Ǧŀԫ>ڐt8)O5$H9!tlR$ub!งvC掠/Cˀ䟛D!D8_Hk>-O I &"Z"x[%DHWDLT/2B؊sv$fwTzkޘi,eWB|E! ͟Dyi;2mE\4d.#<:\͐xYq} ߵU8T!qѣSiR@:7HaG7(MY!yIRW93 !z#@bܔ!u_|Ðt7&%q(zY1῱B(,pؾP` 7:`sԐ-C"Hl9q1_-Fpᢥ8+JU5x=PCw&1ًǏFw}wh*hW*l܁qS6NݭsP{cUU`9.bȪ"]wy^'B"=|C;U( x醯A[ŶwY{H9ߊiĵ٪Atq_XA=8ap'nx6/T!\شOPMcWO*u{bzՂM*h-q/n_P*vtJ-G\1q忶$3Tj@K`^DZZ30!\]|#FE*V k85+B ?T1H;З^ . mr+݁Yo+E+\8l-w܁]^\_ޟB[CɈ-miˡL\̈_U }݂|K4;m?v$"0wy(@h7f7Ӵ+Q3Bfkג[ mgV{g] BY+d%h+@WٗCPpߖp*m@rE Kl糬Z7Qaw`[{b9~42ώOu.=h+##&>3SF~>bxDv2L_#1K,I__"ֶ%ga/h֤J.x60;,x++hslgl)1F;[m{J ]8!^++ DZ hvj.P( kW-J7AzI " *u]n`]v];Hu=΁O6P,}m?WsdOA Ew ks31H?"jKcBug%hkr/٦HP*.qeHoT=.}#3_,Be\'\"%؈}cyty S{\y4 ^{oҍ| CHAp;a0m>V8q򅀳t\ubsT52$Vr9217{?:'K_U!}unq\LΕwvKWwidPLu7:6PDcNtQ_E56lLF)I6Gق:%Bq,ɗt{eW]MtWqkJ%=MWlM~#g@Ce \r"l:Z|q7qH-Pn5=4Pٕu_Fz^OGN޶tב6SKo"cjEߥz|s"2U&agU:0,31^܏(ƫCn^;TWQ$rHo))97ŮGJ0niqQsX9 jXk$R@*[MU ŋ0>I6z-ʷtw6v(Oe0 y [&܇9a׭?՚ѕҳl=ڑ.ٱ2@مb<UCY|!$ }%G 0>6a,Hs_kaK}FJiBDa1ufVI kއ3y?DmߧMLΑPwq~}Ut<[ܛMZi |b N0Y阂\03*(;_g Dy=zz':;9@' ^-ʼ.Lb<Yx1?2uqeBtzVVUk231 !NkH=(lNLfC#?JxZbS%4s\L0ߖd9Bk=ޮc^ BS3|X{ߊJQ9k ia!7PwiyŐ_f ;+0!ƠP5j cӒX(?\ȚM[ jQjqo'e G:sf {A1Dh[i n]c g!>{eAڦ6gj1c0!s*űSK3sȧ`Hc<+ 0<08g.00Oޮ$w@l (R>jtEܸ#o}B鐮&'4-'/N[QK*$ݭ78J%y]F1*S2NW0F,A9{6=[s+O)7ŐxxB@a;s\SMnjM!I"R ox{2Y[y.yMD΅7=֑YV幠h̭"%{rS$~%E>ϳ0}HxC\OCfo 5Z 琉wNǒ/ 3Jo`TIL3t@hJ$3g[z;8Kl̉}l/"(. F䰵~RDZ/#3L;gf6gnK ؑt=Z{lk7+׮t"MۮgOߘ`$Aph"o ùϫ{㈨$姻vV\#u m#;eL ^Fjt ˟Gf}>:D-E+.^|ڏ꤈?'n.cCۯGK/Tݥf I$ڴE髭.-kݪt#|E* >5fν|p2hT&j)/ܴa[T:DL6ż|v(VB'퀶SBuS` eAS{^gu9i*+OB+70x)4o8}{?6OٗڨyL.|[bu,A榐v\E3{sk xB*0uD4n+*W1{:IǺJq_0 rtH&@| ўWA39 0 ys{@|[&I1j=aAU؜psu_`*O6CkR )* \sV7]5ep)^ ]](=7&Q4?Aœ7:A%ib%N/{h#4s笰k贯^5m?*z# }^Dn|ht Y?wȆX~"[z%pO~6Ue %17'T׶\` U@,y≜-=^f Q+;N@ͧ&п(qŵX+=KdIh"P\ؠ]>E!*4k/Q|ڪ?a".?kנּP. #(>O+ܼ)Џzcj )eoӭJ[5?/^gcs_E>d1|Vбlꇏ[@2+J_X/蠂}z>;%8B#_ݭApy9"S5@2|=\й ur4+/H*h-dqBW%ݥǽ2,o͠yCH<,ބ7TFkZ)/vTB9WB|)S֐W2_"E0xї Szٶ>z\+ـ AS$Y.hy9(S |TF9d&_tTT-K?HWdYB٣ 7:gd[VFeT.)KAǧƿyε0C4r[3j˙'@XR.B 䜝PqvHZGkYx-HW]O_-HBK=V?]lɊtIh;W? <KBIHH͕Ta7n[dǣ] ™umh yǔsq6E(&CԮF/у:pk y@$2/,_WdHyplc.d;pWI)" KGaƦIT~@:qFQpj&Qrlm/ڣXY1l~D?sjc~oJ ́r9f0~A; W7tQӛs~z'Hd6QW'ԯk1su1_8wr;ͨP7| 16ʍut*gƦ.b ]t֟9*ed!)'|*㌛my#f2c5a71+}}{0B]Ĥ>pr)l *ʕNg@ۚ؇(Xl-ށy,$1d|{89M^3[&_3&/)U|Kk;{ZTrͅvaZ}lw@Ǿ0g<6EU5xW+nj8qڠhg\_y4QyV.oj5*VJpn$HF)ul4Ӧs -Xv<0_D%.zrwcq߮[MZ.7`sUs/Κ7e~oRQP7I`|na9pn*Fss/{h6ۣرou͡9bB;֊3P~Cbފ0DïLÄ׷DZz~Zjivm?Y} !dqz.),l 8矴x{D[ a?{@Q dMSdanQ@CPc%K33 |؟ӗpT́P~Lɭ;%x3VRb3O*)[晃`=˷ c.Ƃ\vr[cFZi1VkEcI{62̠^rNgsdU^?ji/5̀µ^pM2,eӍ!.-d{Vb?jW'e4I'`Eu]vs֭x^KE1.S"c'*swcgN9W1D֦ҌY2d%u:N*{K,(Ug?@;%I߈Gy'#XELz:ցj\X uU]Zv[;AGgmH`d>slo%ޅ\r?=FV%F0vB\/HlAUe}~5}=$Ϸ_7qt#zZzˬƼxR.a^S1t[W^MG;/Wf."TK*MMGZZSym/~^F?;߻{VDIK3vc|>=YV#<(WxON?j>e0Z`];7ȗuQ]h^ʧ ]_ok31u4B_hJv|+SJ,J|1 uOhɟ/"Xu]t$`Fu}3AM u3}RQ-pT=K6?LyCxwV!=#$`*Wck2;~&ңgkNߓ[u ԭv5˘j|_Os*E}Y(ϼGTV#;|Tud ?Q[֭+I@\d\t~3Y:瑸b$(:@k\!+t;Fn%:T:[~tqgL{tuJNr#bpȀRi'-M"-okz0cQEj]vMn5uv+AP%xփwTMѺ>Pj-aAZCfR \磋A@.O\V1k\P s~|p㶂4}b// z6_a .DO6M@{+s_/w1Iߧ -07$ҏ=jA!0Y hxçǦ+s%Sș cyY[&E.}7#yqhdWq$sM&9<$3 ߫%fq$L~1ry_SVd5#+60AM)lfo?;p-] &_K}>f]Vy ];3P^πh(泬cj>"GX`>D^Ff:4@}]F}ތUet(ACY?'EzN.1){1o\៺.h?ϗ. 6\m8pD.̿`xrz#&BVyU,Z@kٻMO\>,Ӂ:L5b{7uNU:J31)ar I[}Q= Q_5K0n|b / ׁ˞菸#m:@ؿpwz-Vz;̫J:ݒsQ -㟙<~?W qR$V!%5(/m;{e'㘯eFP vAw=ӎ@5^+iϪNj3~ՙoh Is -;{H5g#8:G Wz(ie"/%Ր8'uj^ ,$\z}s}^ImV$ֳ4};?T/el5/ \`:kHH9~- XAta4ЩM%oisGoMjY]c=W(- kyx~!¯Y1u]eM v'MnoE#Y^fEɥ F`#Ry@P=aߥhZCHgV17J ~ɞ-ȹ~q3emHsk` t:.H!(>/zt᝶2EkdyȄ|Il%S(o$Ar扶`d,~SWZyuBS<$NRy ]PwMZ8AftD^1Cg'#H6 %ʭm<i %Lm^VՅAd"-a.Sȳ!9UfB%H,%fLۦ^$Hc9D ?p,3tM тEdoҶ "zCC}‚1jJ~'yE MڱiϮXNQ?T1"?T_ZDgf?0ɍZB8ݔ'ȯVDޚ͹ m_[i $ޛ{s7 S/P!eYܞ'CS7'@PzvX(@$(g8nt %xlUc)|},3w_yxeH*Բ{;"З^ ^k*|ՍC Mc7]0-ݺ$`yЇ *}CNPsܴ!mJGh*Pxjxf)[*D7_~~s$>5E8QEO~N;aaP̕bfYl9(6Qw<oY8響 tn=?_C6 ,H<| E&yx͘ởVikmkTxʟRTq7F g֫CJ[z?l~!S\&=zh\V7?IϘ, JOǽM85 =s1-Is+c~-1ko^'Qj6A&#znh٘+3@qujlG=B9rgiB<5lݟɤJ~V*1~|=Wzϓrp[ECX(Q~لӲq b w9\&(eZѷj'Pn#M3a~(w5!=Vh8u?ǃE.=S^lʳ3RдN|w74}DWƽhƿ/iƿoiƿiƿiƿ/jƿojƿjƿjƿ/kƿokƿƱ3g8K}R4鲷O D7{MPACjK.jU`nhsk4FUT!N\E+ jF*buQP_u>e3,Qvƨc/ ) v@yry_4%їЧC>X|:`oNUt%<Еe",Z N5a|~>;[Kr⦟n\[ vog/ycnB$bcgLKlvk]wFX z3HMN.%0=e={g/N}Gզp}/b2@u'˝c%Y/߱@IM3|,Pvb6DM y^^ܯ:Mk\#67t.P~=@t͓Q喧2@& J̀ .y" MњfU^Qݰ8ݡYG?MS"WiƖHi{?̀K~-3DT!slƑ] QNLMM܏c7p?pWO.Wh2D+o:쭆HCpt(+[R^,W娡 721(哙u4ҚÓ0η鍐$Ѕϱ8U/OlGno1}YiOs^95p߹퓚Ub50u@q4DK}6d!Q %^Rx=bMHC2g#[D՛C|,GaN8S>[Ư|п5=]h2L ~X.gXwhFuH͏<^oX'uܚOӘ_21X&,j\D]~zdj!Z%ޫ]ZB 64Goq5_pMmFy=+XrG&oxSOG,C]Ny9+~mf2,3JWCS2.9N8> ,1F8K+P1e(׾s?{wvfqtK`~s9Zџջ__W]TyCl[}J}1#Xm%ۋ+WW^CM)>Y)R{ЀuLwE"7e_hU{hOS8~mP<'>Lϸn#&z"݋b \ ϻ^tgcScSO5ɠ 73zK˔IVw@2 )s#G{9L3gi=  sTJ[0`yʶAqs͵Pqʍ&fAm5m,DY&/Ѣ$8f?{0YtpCp(A{Ǡnq; U_kOe>o+>;(He#X^^^{dOby+ϺRtzyXʱsEO"_So"v4WyXK0^Ү^)֟ߺAKW[# GͻۧyZw`ITj^z6ͷo/>wīf ~юi1 IJ kH(S%/\)65,x&pӃymW`cx9~umOvuɥ':0P}}pql@c]feȅ,9?BgCfP4_G?Zh0t7_}Vcuyfq~T [ k3hFXf+~._ {g6v4DxHCc6䟝 pdˆ)`)e_Yv0S(o~igpvrOUFB4Zt1*؏sԍ 4#wˊ11Z0Ơ$="\q1T#ˮ9Y9qcn:t,zJomsuTuy1%'oeHW@Xa+<ˊozFIvG}vu͌2ĭ+灔[Ｍ xˠoz맨2n灷8 d>`o nL MˠS@ٶ1<$>Uxv1(v&_`-@ӛ=e}|~)=ǫ>Yւ9V^NSkw99̓m>DgX Uws<ʢny__#&9N /, ;_x$<ڇǾhW%[%) Q>)7Kvj.WiPm%M}qܾNeػ+lG0ҥ$W:PB;xި{yWj~izsB)`N a2P KfKNKz^o,r1\q-Pb [5=_Jo0%~]kFK,PX"f: d]%՛eQf_k {|%W*AšZ;ʧYgOCbM[Ak#{a=d_L̕ A-W<}5OV?ʇ(h)ڛ{K|kOqDqЕ2+S j8a`-gk(ydNp U84)(_y=Ycc- _}O+b t$ylEm()\Q, KaJ,oP>R8]r[܆i}n~L$^`HBQsaEiUKa]jY߰k荻Ҙb͏'߄7?MN$Qm+E\ӪP>6QCġ~Eĸ 8b#ܓi0En>޵erOwGBVD n>A)]xs<.=Eо&9`l Q!9DNߦFkINCA9CS uuwv]wv':o%yd̚o_b}!H1mc1紭|%kPlEnss| (ըZ/ǵ}/I|Kz[?tªP^ffvLe rXs}@_DulUAH7G :YQGo;tC.'_CZbd01MtE=K #;\< ny8J{ϥCy/Ro]";UYksI_ÔA[0qWP~vfyO# ~jZ#<^Ĝ1f}v8Jd-g2Nw V{ը6xvw+]^BGϽv];rO뮱āx Og,#^rㄫ: <Q\-UJGV0ʏ[&d&"8)`bw?ű2!o/%oE'Oyv6wCq5 V"xA)}5S}ѭ}p#ydWao҆ y}Χ7zr;8׆ %\%yɀ_-3cͥ5W}J$IY_*8MF.2bד.PX|3[Dtي۷꟟APZ=#G8`wOash>ivgU[X[^CpE/y+hǜ, SYcpwWOOJؖ;r!Wk#*^xIN^ta[_Yw:H)v#mk)8+)"n~ 30>9kH]s08HW*/ R> XI5`ofa0B$ֶrOۿe[Wm8Q<im1 ꛾jloF6w|]/lqQ]2rfAb&X?3,[ ql&Iy1SWNH]ىIry͓ 2l_pC`]]GLɲzǶj*HJ N_,T0gdes5gUč &[F?87|\) }A2Cg qBT\2?P*[ }:{/YOy>P/~OfٶRL?t~؏Q_旺a^iKi;NCdnX_dS"d`=f?%.I]r>Dg;(k͵Za}Ͻ5MJi c*{)N=Tl)O~t[WnL_LwrU5A~ 30SdA&yR^)xw8AĪeN8; JזYaY8wOgम w>G[LC褫vJv: sfnr+_%{~*a~ cV?)H)3F@LWȢ'W#W6[~BA JiMӻBGςe QZ}O+AfKׇ XSx~Аw5zp7S3zvj~KJO,e-8IHm3۲ۮV tW=Uwf3:oZ˵jh(V#5:`Y_b9cJ9R3n֍1rg̟o=\?\y[N?vm3G Owi'Rz̐vAn`3f Xbfb7hxsh݊^&Z"F=[|Vks'p-P17o @'t*qN+cw?s̆6)WJryA1;ea v4i?Q9 \[R%x"9Kѻဇn9bت renȣ"0g΃5EKKo՞媙 -3r`=u7yi8̅CIkP򔏶5 gP=P ![Ԡo1u93Tu늘l:mT!/_GL;?uP3tws8ƥ Is^}WdCԝÉXN-|ؿk4"3̟8>s_m3EC9*\qv3ELu#oka| _2EtNr9SSOZW"EO'LSEF׎tK2Eʹ9T;{eMQU'FG\ß=(ۏ1Al!!^;N w׹?6*6&:gGW |i&赆Li :)Ȑ7qCĔLW7oLm[Mh 'T7D2%0i7&(tA8s-&}&*]t{Pka5`Nʂ ޗnB0nI1wct28mXƨqm" ScO|GH|Kp|d/e ;aEYqͰwۯ~/7v_E'ǁO1;6tC*}f0;`n/vMSOaymޣe8ݬ.&%fQnG0Z_M1@[ as_甠!zc^7U}gz!jx't* -vW3L\0ޝ!{ \4)nZ?4By3=ₛ&K6B =~iUJ;HCMMyWLzj+<#n\#,_e ݥÆݑ.#Ɉ! J(f-\&m+5{8YͮޝeU{Y/_({T Q[W$Fնn8iz5h?j=g$ٳG 87˕h뽁n84>Yj`9.%XfI C:}i_!U֛+(`ߛ@5M/p~*9;=e`ϿpQ2~ʶ]bXjwLo_!zui-2Dy6+}'=p#a7)C%!kZ>Q Qy0u*P0DUJUtWa?Am«5纹IEݽT7߿"r9|S2P%2Ǖ>Z샙G>l8)>~Igs$ږ먠,lt;2Q%51qqoH1\7n9w%j sx㠇Į̴D /㾏ޘ*͊nYs< iKWzc'LDrzT_ ,Qgg<=A|xyu[3_Le(IOWȾs+Y.oy>pݏޅӵm yb!L;O<{.dNp66?F ]T2P#|֦1>V>ƚ9J!O7F8ds>Q'R%8vj%|e} ZӅ}?(|$a-ޘyԺ*VS.V1urIʉRhOLT\m#h+n;&2뢾 gV3Qգ5qоt+<*_&J9yp}9L8rFo\QL&IxaO1|} /WC|hͬCʃFi> sS<8{[9 <X|Q'\<xk}Oܸa[TيpzZ ܨfN orS}=Ny)j9s 7n~=4_ G+X;8~y;7L;30k*'~A f%~xWw8hښfh_>_nLxWqWuz7W$R`8-(:}v\qKCN3DQ{8o#LMpqnחI;fܿSF/em!ۺ~2.8؉]Էuׂ군Fܸ^_ CnN[׾~os PIdVcP٧vuƭEpkJzЩXTb*Ţ~yw bۃkPr*k=jrIM \7t|x%fߎo?닛̣8t@*a✩>.C{ǩqP D=A6uҹ8өfEz&-ac&Ϫ拙F?p/?-R~gcN9S"G_ZmvY-yyOXG=lvؘ?!zw7Hv qʄ#ues6ӿFpt0g [Z_vx .w{D7(9B\0>r*$;/eeC9P_ӚgY^xh[ ;BѭGo\b"/# )7rY`ҁ6UL\J&S[2VS;  /*ͽy2$@BH q rR`7=~ǜ,i}P$;ZNc! A3,T|ժLڻXsbY0JbBM QaVSDYs5s}>]N/imd!=$YBlI XӻzgWk'K~sv!^oB}+;xѴk- 4̗_PBWܗq.Vj_~nN([A+T/̺1MK;Alxq=M`48=TĀŞcαO ]@ϙ\ܟ{ۿ_*S n{ERkStQ1s%W,>O̾m^i {랞@\$96"XӻOGK1#f`@l(9hv \X(2K\m77k{7G:ep:#f_0֬-9*_hc!qqiFfy=t#=yV].  Tb1x^b5S\$NxbI~\b)gD6 /翤L *~r?8o }gCyKMvs&/%N8x*1}mbġKL!nHtsln=҉˒3LUcYqYĢKUԃ:ELp7o۳t*^jV˸r"io 0Ga0W#XCzN⑅G6"tK+y$pAɘ"1/ o&|,u$9x[xbrJ bnqv}t=='i1t1N= :1SBܹq鄨jr{Y17|&뮵!.,<Wc|)i_^"Kx@.k{~r@,0_P:齕_\1fabpj'1MDďS; Z8X4aC󘙇v%/3rR_ҍ2L+{>crx'&Qо/>^pt F=da)Ӂ/7onꦺש%Bcj۳T0/s \ @F]1fExrI|51?r8ibf `ߚ1S6.E~7bi_s`k1X3ȜRXOiӴӝlWKQN]hqXdJ6q=C{Ĵ7%gWHVFL]`;yo]Cpcs&XpqϮEZphwpɹj0Wxezi{ l#Zoo,W'oyv>=QT x$5s:dqqA(t]Oc/;{?,jwژՂÉ?2u3U-+(I5ns{{5",bڻԢLꙏpRb|{9ɱ>WN&nH(R\ z_)U1o}$!qDj1Svq'q¡]¥kX&,\ E=Ak cV3 f]3<6QW|kS,Η%J&n>lP|2ԋ kSCD<mnYčI^*_d?y9߮Ю:y~og?X#qr?qٌkbR~=f(r`b,+ϋۤ6X2p53̎)aۘMw- NWI}/^qA[55KO ,qٙFy]4uu|Ubڋ/_#f+!>Qm8VCAW<s)Ko"hF\`oPJ1H&ǣcl3崮WZ 5noǒnXz6eB<3uq;kzS仢s[a@Lk;Skä׵,=@:f=fC_H9^8_Si))lb#;ƦoM@u&v"- n!pC}T|U񃇰L #~-sQoҵ(T^GWXT~!._}QՉ Os}F{hE,uL[uF6ν[cN}ILnYD̟'=,Y;:fݖYЧ͈kDܸG\X{l mNiiz&s|T%?a틓aDZن~3ӑ'[|b[]߉OD^Yz-dӭ` \#cyk!.n .jK**o7Ӧ#VHq&WJ:s揫kl0O<$̇>ց&/,Cy=NCm5?< 0$gqo]tb;dbaדc}xi>τ?rp17z8f _NUO$xx;ĨG; w-c\G\|&{>Xmɩk)_YR Ǒ眏SokW'k5qm?2!ݹ[ZWSi$k76<_7}0;r&N2~O:Bh@F2ރ'-DC&qU1mK.%nw5vt\]TwBE\N8ۈӥ|fMհp^>CsM^B m`_a+$ {4r7Oٸ֣EWe3Gj }$ktM|E,Sv91UOGpގ1%n$n Pu9h<>B1mYVh5tĉ]J7~ţ}Th4pR}=F`~|q1"*%KE2}Ӿ״hOD Q("4T!)B! e=Rzs=wČ|ͿpT# )U8{~kuR#_X1&jS8,dd*M/Na"3GO6ZA@P\?4_ C:MVkk`<|mT<=G_-ǣw G6sd0Ʊ+|pG)E5lpR{/8<1z8s֪^lwO7kEL;.iu)!W_}k]̷*N?T~`e ghľz#6#46bo9md s@:FcNrǧXhōg˱VypJ)՘-{l Ų魈 JKm%هm#xɽ#6 !~sVggykB_.5m4[PwеׁKqS3eMb{8+F>oU％‚3]M(FyB=t_(teyVpϭlן6wـ[O|2:Ak V/U=|c) wҲfl;iO Tp7×8`+7=KRz6:C?-1w- FvIMiy;ڶQfKV<*f Gx %mϖh_R"(m/O%zGXTgYyƗ0ߗIr9<%³ÓcgilP.<[G' ځцfM~ a>kUuȏ?ˏb]KJ~DW3VG@Uj˗{2 <MVhsw5~Jlv-~$8XLv^;5+T׏lJj)LRr{M[hY(a?5?59(uI|-8}|ZORs c4u?`h{d7tΟ$&6uD 巙ut Z)w'h ;|7O@MiY}=ϕC-k)5ve)8iӡi+HkXw.U/Pn a67|5 9EYY!#E[:yU3xxt?6pg6g=q?ii ܭ.qҟtk-n7IݙM}̏g٩l)ϻQo-|tl7;ob >?\ffrgߓlJ'rXۂrKDpM=if BS ՕZa ٕ*ꌘώXg~O"ii @~` 2M1^Sb87ʹ:о/@6h:}t# )?mB=tjļS 1{bs˖KK09RJ! eh.l v{䳃s9'`,[@cd3i/M-nm?x^']6>?), ;>r ';~#_Z^O,iWl@Pl{%?nM^i6 }RlcO/_/V!Gl_JR6/uϢc)V y۵iDgEP$D/7']gX@S>Jz$3TY'$Xf6!sN/\xwePǾsf>-v *Pu}j}MeQֺ;r7Z7*ڸ ӺtŰ${ڗ⧻VuђKRK|f"kW?mw]U-o\T'Z%:R 2Ω4;GԽ L;{?᝝B* 6_} yG̶mL?3[9I.%U#n^RzV 6X/vlqPi #),I /.fEJXCj䋾߻/Յs0gcg 5`ʛ2UhOD BBS@}SПu ^YCNDkJє&TkǬ޹#\Gm_^0/]'sY ]8i"s(@(wfSIcfv{s<_LU,|G|yDﱕ5sMCBp(2|Q:(YO_?Rֺ*+hJŸes+ .i_"XP͹$yOx~/v n'\ d[R ?qJ,8_qʾ6Y~q쩁VyW}W  i-Fz׼>)V`Ghv V]:Y_2@Z*x~ ?5({X۶S}U af;xt m}v"^+0!9=6,{<|ZP/ "ovZؠA4 TZB2=/e(Ue r{w=҅PSp6Յ,$ޫ ǦFߘst'P())9zԓ궤yH!>Wqo8@*yWی,Et,A,\> 1o@NreM( EVk[U0E*m]<% m*n9zuV#rRo$sw'+"PVՐoeaM|ǂ &뱵qHInFn E&\&<АjjCHpCv' ,csOy dy3ƃɚ!a%LtІy}V2$R>od4{N⣥))jKON&itUꛬ-IHHJxbn:ՌveJmKXw;E}HЖA EK/Bl H9>ހ4 wߒu<ӆ0,{WIGцؾS' W"il) }21Xo(S^Fð5H]GAs:'KGHqQ{Tg7 #QvPhW4V"\S`.<Dܯ)ǽ٤GOM3xwI)V|-I+sM} HP~ygV)pҼ[5k+R,Aȃ?͐ލth6i8@ʜ8aVX,;.&<$ p1C"0*Y~q|#qrK`GS(.8/%ΫZ,4w_-(zuWLbKs pëR\lFr89@bmM -CNG`-ݚg2_20*&@X;.A,fept [|ϩ3 e׵j6dxoz%uڧL&P_lćL@׃X(mw/)P+mrXYbƛ=ߩ[b՜o6&߼*jGnYK0a+YꍁTkCUê༑= ,O@G:r־,GZÍR;d6CÛU@`ѵ^1ޟI{wVJ"chy-la>4%f~0wA ;f#YwXmS(X}1g@mA :0^jSaЫ|s:K>ސHwxU =c1 M_c UNڃOc85f&]+C9nGL2ڳ9r>;慔1x^6ߧ5Wv@x 0m&2%-:y-LI<;L]| l{B[um/LlshBM m٩@8?!?>s[Ԧ8~<#-hVuW=w\>Qh^1,E?WCp".]58Kw~)ΰ:U&@U}` dY~U/G_!e}T4hZ; *0}u_4lzYkd[O٨\d#a0}) RPH Ph.ؔ& _"8 .um~n~ץX>2gE;B #򓗣6ޯ1vbn*ixC"~f>矌6gˏq[/KYT(_Rp?`G v/ڵL@)z{Ld@sfHW[AժÕ2@MPveV9.c=Qr͝c2:)#kc&rq$Ϗ4`^1vj~E@p /|Y:|jq D4]FQG'ĠU$h?ȡ?<2 pr5q1P\F')tAH&_u\ Z3" m2g{Bu.}e%[֮'Dj6qjp'̳=,!a폝=m-Š{ )ۣy?G\.?<30GVSPN.Eui9KwgaA,aiBg47lq;y#It{fBNYoU>DBPQ!U[41'w2f| Yhn]분>+}ȀJxCn ʖG\}JquA%?̇>V&;V#RɕX_Iؾ7Н"xz ҘD#;x-k*4|ȿAפ**gh/ܾ =$+ H] ]sB >#K@N/ Ms{椐U<*y :c2S]G^+]p(ÁPʏ \cɸDu.D={lMHH,)!uk?kC>DJ*oB5 n"0*"Pr}pz@(tw-G3R ?YɿZ[">jB[5=K%!|zoo H0 Vs^y'?rU l]HX2'WaHFL,ir ¬ V+ >v"#Qlw 'OEEW3^Ł;cA:;B%5jtCW $I@<xa79t}8?1 9dB$d ѷ)pI)lϢEPc,F,B|f VwS@PہGOX'o?~gCݢڃ|:XZ^%c"|9cמn ^ٞGzG,d=$c}~|0Ӱ;?G.=:fȞvi޶9lXa#~.b_eWW76_\J  k<% ڟnjd~Lgp*I@w}۫(O'Gkm5 [r[b7/)T^oR]AzCk=q*{\+ZA'U*HMimUʼ{27p] WJƛM-Ԥ![JԶ9i;qu ~[eH//!8Fy_*Cpfdx8llm:7qQnxO')-BcIxV1J+Zv2}6Fُ{8H"75#GpElN/ FFpedSl)bw/k2ch.7e还1Dʴ7#*:_XA)yTeO_|d;i]MO$>M'܁9}sux1[=Xm2rs֨%kݡA5xI4"?YQ;\x=!K <_֝p;}v,z ՀNT>zW.AK\!'&M+u3>x/#*}>SLWtkH*@^iن=l?U`M@ hp 末عϒ[RnXZ[LS@V;Prei\ODLo/)k@o+o6M}ӌ;$*[Mš_O3UALD>`"\UtQSHTgjrI`h{k^'Mb; ᲏F~&E4T1p[67r@|$H 7kfU) }`m>a}7ˬE-oou6ֲ8q;q_>QpI}M8WV1uZX8'%+ vT;qOV+}D{Wnfn"woSC|\+|P=5B᮲XG *lTCm<8o/bx楸aQ\R1DɅ/b_7HE{S3BY1P%$l}gp}L?M?#0oϮ) E:88t;d=zfFcqg[q({!_b{{9Z`;wqUբuYC_>uXN3zq^zzZ[ݵEQh>NvpSl%:m. ?c~(q ןj1\?$"Y#==C{'oOC@?}7oΖl!ըV,@h]Qgqwy<}t!v|Z߉zcݤ%wmnz:Z"a[ QZOAez١;#~$%hvǭhO7מ N*m)ZM+VG㖄 aגN1i024AF;_aWM {zNwސ0WGCCeۺdڙ$y)-sރjw%*?x3pS,f}ﻥJU@e.D t^?ΥevhXm^~fUWl|'U!M#y;t/թ 3o׼_ ׂ]Txcϗ#:Us3naV(S+6Vװ_ 5)^Vfh` !Ry4\߳LisYuܥ:9ESfOC^9u{ՁdCu]'>~Y͛%}L8eC˔1qj\TpF49~_ Jp>5χoE5`F7}ne[nV2yt&G >#">#xq?Pt A5~@`V!e!jO+kox P"aZʄ7ML w^Kaiciʤy(i[^T$-sV _ MJI*+Jn>V"S5!HfNħܲFi<׵a`nZozi:8{ tW9GHt/ /JM>N;ʁ>H$-7ԓ7ŒFR q{K;J93dhmv [ Pkx'˫\8Yj3E\t@y)H/k W .4R>:wOlR$K-?U6)2#_1~1ɸ [/ C,H;q>*<$7LɫQTˑ>?Q?EDbɞfM k#ޣֻH: {e.G\Ϙ" I1y@\ϻVԫdi ᳲ$uH:5Tv|A\Xm|TrWMXyХqmq$Ɩ)K9}/'kDoaFl  /<0 ]!C`{bLzU Y2AԐk:\ e?#[7ŚOAG\#??f 2'CF{ޑdPCEW<+C_6b "/i_/2Jwa}H͐!5|{>sЉP >"KNˑyNĄx4Ѻ\.#LWQ6B~T rfPb 9rjbBw ;%G Ҋ߅}Uj5brcErЉUĤ6|OGP}-9]e,3劫MGo@Sڮ #eM;͂uv2iq;Mӣ)#oݜHT ȅue/|-Ƨ!b͋f`wuGdx3>KUYC=Da2N]sd̙6*d ɛ٨Xr51|~j{PVe_̌GUUh@}^!f }g|5;*K$ =QU~?.FxG`% e7Erg0Gu^*jgaqL,r؅:>{[ZC!ي/:'~Y~7mOZ'r!nX=GG_j73iC2[Cٿ}׆dָRW` ,e.K4>Fk`ȋk{o;)%CR+> 1/!gW}ӓm?xadRB'g`Mk:_##!u}(bJmRļNW⎣&29MXӑx:& i8-%;:1mҒ]}V/PG\šXn7^'+y=B̛d-bJأ娯v}ڗS:'W6b`E1AL=}K3b,sz@'mhޣQhXHE{m9ʩa.t ?_GC 41ĝ盂h ]1?wjC1W?e҈Y-4!mt)@3JEL=ο@6X|罷xqA\z`Ÿqr8IRͰmM~eS{a~f,6A~au\:i#G,BL;@1BdfC50h:Uو~Y-qaa. l-8Qwyx~_b(aS5ָ)q7`#Ĵ+,BL=0P]UB?dcabzpuT!穨jOyS8b{~E’>]lEh\|VnGA%eM0BTe1MGZB T?F%)փotv>?.ȎJzn~ 7'> eD>x>2r8١BMl骿6\F>[;ZAeVt*xץ 7]f}fn5~bT5jp>AR#͚ a5`mQ9n%kk=9o\ ER2N}$y>u`F#ظZ:=4ӕ2Cf3ChX]`wu(>ߤ!Я%YwI߭5R*ыԡ3y]11cuT[3?FZ;tR2Fy|h^G2~6:[D.eF6J5>wLž[;^HnD/QͿ (?k51|F\moY "H/uE]G8d'k"=W9aŎFБg3ja&'CҭuV!}cL7q8N'ΔgΉ3}8q'OT;KJ/O#Z9*+C pN6hx_'`qZ4oMXhV '`.{kpA , 1wokK~>Q'Zs0G`eZ @Π_:1TإY%vi|?GC݁sQ㱺V̟>˽a2`Nx| /WA@ً0Z;nWwgOس փcL,>|G/d=v⃣{ wA#uӫhbޏ6eT8irJ)~!x$󆔦g(v4[s n^֍d{נ>o1f"5sߟM~sb+N- |i}Kٺ_fPrY2 ̠oEl/eAfՁ5K̀=QM?zŻgU_G9w|^ܫD~}l43LiyBexFsGS8a3a:I7mOm7G䳍`MziؖX :-1Jce8kDy㲣X^wj6ʳj7(?R{{ތ5>WlHٻdCt ? Ϗl[yGeÙzOD~Tkt?4uS*8xH)A+M&LʣD9^24ģJ HUƥ}xz+m|2qC|S$N g,7O xV(%{d4W&Is%>IH:FbrUG%%"'!@j'u?_NRč}zt r/CVqY|. gi¨n{NQ`-vP T7b@(Q7j+%~cP( B9-&GFI.e>g]<{wd]諽5;*Rao]_3{{84>y'Ajm5M s_{+ XEj$"|䊚#>Y=8ngۀg{Ȏ 6|&{W_C<3ez?kgGݧd#9 &anp9b։1Q+{<&d]6J,&bұ1"hwxxXo#HKR@(l mZxd7>ZeP0Q^Һa1{u\QF{e}; sirt \AM󦠽[*3b^̑jpx0 mY"%~[ZE[Uu:5Józ# 9Meaf!$jE,PKϜVj7d"d-P8! ?ޠjO3ՏyJCJc"}m^)0L{&߽f7Iu_2MG*ܯum7櫊 Pw#+7AOΣHgE]IG־Ή3_YRN-H7^B ݍ"̠R{P%h*E4P %C7( \]9?/V #rcV'b@3 *,EhgB~)O/ԴyP/9i3w[s`;xRVޢjQ┴IأtH58^i[" yF@1*ߵ2 Ie\Vaia2j#z6G|2٬E~ForM| Ӵ;ڄf8G}B=|(Q;kO!7bQHl%5Zf96:r~h ^F4ZYL4ƅxdo6umVF,jn.MOQsh;}&=F%UWfM٧ Kpa2= {y5X$|&0ru ¶ ػ^[4iv]Qγwꎪ-<1Ȑ[yl*Ń \ utW;5vǐPNP ľ0[LCHx ݻ> <𽵑1 K+dVI92wdqovRˍzuZ i*\.`Luh p:N aq\H RMwsyFCC6fHњ}\P&MWVͰ>]]Ѝ|i1{00W.!UʯgŻ}*@/[hs/W"[A\©&)hW>W{_RITk, qfڄWI5]~̮ISz%ڄztN{:4H^"O&-Ԁkgݵ""#:X͉d݂xMh`]5V? 5ۼ}EZP'#ZQ"_EHW8Ѿ~&! :ص'mu/ 604$$Š` \ 4IT#egPRrv-Mؼ OiZP;]hH%ݗNH>zc=$p/1o.H;cUb{g58o&Քo> ^ΛmV~A2{p-ΖIFJk}Wt ?:?ORǜ8WN[fhG(2yopGbZV"~f@xw嫌Sȿ?񒎴4<.Ovo#:dj'}^_h ~' 9uW/> nBo.'>Nؼ>aRj.y$DezԨǔ#w)oBeyghzt_cA. ju =1W.PtIhLm$Nx Li;홿gVoZyveqQ Q9wfw^W??FkXZxTi]Sսp)_;ꩪKIpnxl6:[=~z2"ɷ=?+6 wهD L :~}Q{w))8V> fQ!/Y+Ekp)/#WX.goiV6s#eԫK2;{hg^Y/K6>׃Tj򝲄uAMq91v@166NN@]@IY2.#(lP8 <*(* %,/C1x<}>=e>*_ZS?֬@7ܹ'zlDKwMqjLqKxuWϟƚc?fZM{::=j8` 64$11/tK Вs\Kà^{+:rDڑjNI/FmyDeׇ~ZO`MZ#q|lF- оJQ)_r7"*w']`E<@ Sܦ`%qtLq+ Nu_xPʑ|.IcQ\OWebL; `5="zϞJ s^ EYc&լn`{o@[>]AGBmoA5qn ɜ?Ҝ9;p$mo?\̹._+@y`q>>BE˹ ^8>A4Op}vgv 7cx0-Jť?/>p0)i1pų}c!͞wb8|/sIkjEqjM-m 2<&=SybW}(s>Ϝ8#dz5z!{];lh/j.iNЛ|ϻLo^:I}61,BNEW1be:4 > p?PKS{kZeVCNTR_RG_60M_2024_4326.shxmU}Le=/H:3TfƜsdM 3>1LQ:e,J9T$!$%$t~kƈcf))!sYYtO}{9{= % nZqj^+㔲3z2R})ܾ c Ezi8 2aܣ Dfr-*뇁ɴy5wgӁw9 &6`n N˻d8s]Li@jF`T 9 ,rqAVųP\_}E~';yV&p̳9s5k? -`C۰عξH{jIܚgaRHKuxE ?6_im5\\y d@.YޗmcQ{5f_8\3{o.HkOZ=ޜhhnӯ@i %OwXoJ<8/&pޣhց&ute#}ӑ!8~nAGV.ƹ:j>0S$:0͚ٮ[h/~sFjhYeyN8 /tx~B߿4tQ}e yɥK+C=O|/ ?u~%Q濄Zq0=<ߺo@Zz sW+ro߷U9̯G8ꐯ M$NBjy=vA{i)C6eD3:%кY =tJbG]nwPKS{kZP<CNTR_RG_60M_2024_4326.cpgPKS{kZ?8R>CNTR_RG_60M_2024_4326.dbfPKS{kZUCy8CNTR_RG_60M_2024_4326.prjPKS{kZ_%G #include #include #include #include "eckit/geo/LibEcKitGeo.h" #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/grid/unstructured/FESOM.h" #include "eckit/spec/Custom.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { static const std::string GRID_N = "pi_N"; static const Grid::uid_type UID_N = "bdc49d97a27e389fb86decd08a185c2f"; // {grid:pi_N} static const size_t SHAPE_N = 3140; static const std::string GRID_C = "pi_C"; static const Grid::uid_type UID_C = "e548b74fa53eef5ab412c6061330f043"; // {grid:pi_C} CASE("caching") { if (LibEcKitGeo::caching()) { using Cache = cache::MemoryCache; SECTION("Grid::build_from_uid") { spec::Custom spec({{"uid", UID_N}}); const auto footprint_1 = Cache::total_footprint(); std::unique_ptr grid1(GridFactory::build(spec)); const auto footprint_2 = Cache::total_footprint(); EXPECT(footprint_1 < footprint_2); std::unique_ptr grid2(GridFactory::build(spec)); EXPECT(footprint_2 == Cache::total_footprint()); EXPECT(grid1->size() == grid2->size()); } } } CASE("spec") { std::unique_ptr spec(GridFactory::make_spec(spec::Custom({{"uid", UID_N}}))); EXPECT(spec->get_string("type") == "FESOM"); EXPECT(spec->get_string("fesom_arrangement") == "N"); EXPECT(spec->get_string("fesom_uid") == UID_N); EXPECT(spec->get_unsigned("shape") == SHAPE_N); std::unique_ptr grid1(GridFactory::make_from_string("{uid:" + UID_N + "}")); EXPECT(grid1->size() == SHAPE_N); EXPECT(grid1->uid() == UID_N); std::unique_ptr grid2(GridFactory::build(spec::Custom({{"uid", UID_N}}))); EXPECT(grid2->size() == SHAPE_N); EXPECT(grid2->uid() == UID_N); grid::unstructured::FESOM grid3(UID_N); const std::string expected_spec_str = R"({"grid":")" + GRID_N + R"("})"; Log::info() << "'" << static_cast(grid3).spec_str() << "'" << std::endl; EXPECT(grid3.uid() == UID_N); EXPECT(grid3.calculate_uid() == UID_N); EXPECT(static_cast(grid3).spec_str() == expected_spec_str); EXPECT(grid1->spec_str() == grid2->spec_str()); std::unique_ptr grid4(GridFactory::build(spec::Custom({{"grid", GRID_N}}))); EXPECT(grid4->spec_str() == expected_spec_str); std::unique_ptr grid5(GridFactory::build(spec::Custom({{"uid", UID_N}}))); EXPECT(*grid4 == *grid5); } CASE("equals") { for (const auto& p : std::vector>{{UID_N, GRID_N}, {UID_C, GRID_C}}) { const auto& uid = p.first; const auto& grid = p.second; std::unique_ptr grid1(GridFactory::make_from_string("{uid:" + uid + "}")); std::unique_ptr grid2(GridFactory::build(spec::Custom({{"uid", uid}}))); std::unique_ptr grid3(GridFactory::build(spec::Custom({{"grid", grid}}))); grid::unstructured::FESOM grid4(uid); EXPECT(*grid1 == *grid2); EXPECT(*grid2 == *grid3); EXPECT(*grid3 == grid4); EXPECT(grid4 == *grid1); EXPECT(grid2->uid() == grid3->calculate_uid()); EXPECT(grid3->uid() == grid2->calculate_uid()); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/area_library.cc0000664000175000017500000000312715161702250017664 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/Area.h" #include "eckit/geo/area/library/GeoJSON.h" #include "eckit/geo/eckit_geo_config.h" #if eckit_HAVE_GEO_AREA_SHAPEFILE #include "eckit/geo/area/library/Shapefile.h" #endif #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("area_libraries") { using lib_type = std::unique_ptr; for (const auto& lib : { lib_type(new area::library::GeoJSON("CNTR_RG_60M_2024_4326_mini.geojson", "CNTR_NAME")), #if eckit_HAVE_GEO_AREA_SHAPEFILE lib_type(new area::library::Shapefile("CNTR_RG_60M_2024_4326.shp.zip", "", "CNTR_NAME")), #endif }) { lib->list(Log::info()) << std::endl; // std::unique_ptr area(gj.make_area(0)); std::unique_ptr area(lib->make_area_from_name("Portugal")); for (double lat = 39.7; lat > 37.; lat -= .05) { for (double lon = -31.5; lon <= -23.5; lon += 0.05) { std::cout << (area->contains(PointLonLat{lon, lat}) ? '!' : '.'); } std::cout << std::endl; } } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/geo/area_polygon.cc0000664000175000017500000004343315161702250017713 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geo/PointLonLat.h" #include "eckit/geo/PointXY.h" #include "eckit/geo/polygon/Polygon.h" #include "eckit/geo/polygon/PolygonXY.h" #include "eckit/testing/Test.h" namespace eckit::geo::test { CASE("PolygonXY") { using Polygon = geo::polygon::PolygonXY; SECTION("empty polygon") { Polygon poly1; Polygon poly2; EXPECT(poly1 == poly2); } SECTION("equality/congruence") { Polygon::value_type p1 = {1.0, 2.0}; Polygon::value_type p2 = {2.0, 1.0}; Polygon::value_type p3 = {3.0, 4.0}; // strict equality/congruence Polygon permutations[] = {{p1, p2, p3}, {p2, p3, p1}, {p3, p1, p2}}; EXPECT(static_cast(permutations[0]) != permutations[1]); EXPECT(static_cast(permutations[1]) != permutations[2]); EXPECT(permutations[0] == permutations[1]); EXPECT(permutations[1] == permutations[2]); Polygon poly1; Polygon poly2; EXPECT(poly1.empty()); poly1.push_back(p1); EXPECT(poly1.size() == 1); EXPECT(poly1.at(0) == p1); EXPECT(poly1 != poly2); poly2.push_back(p1); EXPECT(poly1 == poly2); poly1 = {p2, p1}; EXPECT(poly1 != poly2); poly2.push_back(p2); EXPECT(static_cast(poly1) != poly2); EXPECT(poly1 == poly2); poly1.clear(); poly2.clear(); EXPECT(poly1 == poly2); poly1.push_back(p1); EXPECT(poly1 != poly2); poly2.push_back(p1); EXPECT(poly1 == poly2); poly1 = {p2, p1}; EXPECT(poly1 != poly2); poly2.push_back(p2); EXPECT(static_cast(poly1) != poly2); EXPECT(poly1 == poly2); poly2.push_back(p3); Polygon poly3 = {p2, p3, p1}; EXPECT(static_cast(poly2) != poly3); EXPECT(poly2 == poly3); EXPECT(poly2.size() == 3); EXPECT(poly2.at(2) == poly3.at(1)); Polygon poly4 = {p3, p1, p2}; EXPECT(static_cast(poly2) != poly4); EXPECT(poly2 == poly4); } SECTION("simplify") { Polygon poly{{0., -1.}, {1., 1.}, {-1., 1.}, {0., -1.}}; poly.simplify(); Polygon expected{{-1., 1.}, {0., -1.}, {1., 1.}}; EXPECT(poly == expected); } SECTION("partitioning") { // includePoles=false auto mid = [](double a, double b) { return (a + b) / 2.; }; constexpr double lon[] = {0, 90, 180, 270, 360}; constexpr double lat[] = {90, 0, -90}; Polygon polys[] = {Polygon({{lon[0], lat[1]}, {lon[1], lat[1]}, {lon[1], lat[0]}, {lon[0], lat[0]}}), Polygon({{lon[1], lat[1]}, {lon[2], lat[1]}, {lon[2], lat[0]}, {lon[1], lat[0]}}), Polygon({{lon[2], lat[1]}, {lon[3], lat[1]}, {lon[3], lat[0]}, {lon[2], lat[0]}}), Polygon({{lon[3], lat[1]}, {lon[4], lat[1]}, {lon[4], lat[0]}, {lon[3], lat[0]}}), Polygon({{lon[0], lat[1]}, {lon[1], lat[1]}, {lon[1], lat[2]}, {lon[0], lat[2]}}), Polygon({{lon[1], lat[1]}, {lon[2], lat[1]}, {lon[2], lat[2]}, {lon[1], lat[2]}}), Polygon({{lon[2], lat[1]}, {lon[3], lat[1]}, {lon[3], lat[2]}, {lon[2], lat[2]}}), Polygon({{lon[3], lat[1]}, {lon[4], lat[1]}, {lon[4], lat[2]}, {lon[3], lat[2]}})}; Polygon::container_type points; const std::vector list_lons{lon[0], mid(lon[0], lon[1]), lon[1], mid(lon[1], lon[2]), lon[2], mid(lon[2], lon[3]), lon[3], mid(lon[3], lon[4])}; const std::vector list_lats{lat[0], mid(lat[0], lat[1]), lat[1], mid(lat[1], lat[2]), lat[2]}; for (double lon : list_lons) { for (double lat : list_lats) { points.emplace_back(lon, lat); } } std::vector counts(points.size(), 0); for (size_t i = 0; i < points.size(); ++i) { for (const auto& poly : polys) { if (poly.contains(points[i])) { ++counts[i]; } } } std::vector expected{ 1, 1, 2, 1, 1, // (for this meridian, note that PolygonXY is not periodic) 1, 1, 2, 1, 1, // 2, 2, 4, 2, 2, // 1, 1, 2, 1, 1, // 2, 2, 4, 2, 2, // 1, 1, 2, 1, 1, // 2, 2, 4, 2, 2, // 1, 1, 2, 1, 1, // }; EXPECT(counts == expected); } } CASE("Polygon") { using geo::polygon::Polygon; auto is_approximately_equal = [](double a, double b) { return eckit::types::is_approximately_equal(a, b, 1e-6); }; const Polygon clipper{{-1, -1}, {1, -1}, {1, 1}, {-1, 1}}; SECTION("Construction") { struct test_t { Polygon::container_type points; size_t size; }; for (const auto& test : {test_t{{{0, 0}, {2, 1}, {1, 2}, {0, 0}}, 3}, test_t{{{1, 0}, {2, 0}, {2, 1}, {2, 2}, {1, 2}, {0, 2}, {0, 1}, {0, 0}, {1, 0}}, 4}}) { Polygon poly1(test.points); poly1.simplify(); EXPECT_EQUAL(poly1.size(), test.size); Polygon poly2(test.points.begin(), test.points.end()); poly2.simplify(); EXPECT_EQUAL(poly2.size(), test.size); } } SECTION("Contains North pole") { Polygon poly{{0, 90}, {0, 0}, {1, 0}, {1, 90}, {0, 90}}; EXPECT(poly.contains({0, 90})); EXPECT(poly.contains({10, 90})); EXPECT_NOT(poly.contains({0, -90})); EXPECT_NOT(poly.contains({10, -90})); } SECTION("Contains South pole") { Polygon poly{{0, -90}, {0, 0}, {1, 0}, {1, -90}, {0, -90}}; EXPECT_NOT(poly.contains({0, 90})); EXPECT_NOT(poly.contains({10, 90})); EXPECT(poly.contains({0, -90})); EXPECT(poly.contains({10, -90})); } SECTION("Contains South and North poles") { Polygon poly({{0, -90}, {0, 90}, {1, 90}, {1, -90}, {0, -90}}); EXPECT(poly.contains({0, 90})); EXPECT(poly.contains({10, 90})); EXPECT(poly.contains({0, 0})); EXPECT_NOT(poly.contains({10, 0})); EXPECT(poly.contains({0, -90})); EXPECT(poly.contains({10, -90})); } SECTION("MIR-566: wide polygon") { Polygon poly1({{0, 0}, {361, 0}, {361, 2}, {0, 2}, {0, 0}}); EXPECT(poly1.contains({0, 1})); EXPECT(poly1.contains({2, 1})); EXPECT(poly1.contains({362, 1})); EXPECT(poly1.contains({722, 1})); Polygon poly2({{0, 0}, {11, 0}, {11, 2}, {0, 2}, {0, 0}}); EXPECT(poly2.contains({0, 1})); EXPECT(poly2.contains({2, 1})); EXPECT(poly2.contains({362, 1})); EXPECT(poly2.contains({722, 1})); Polygon poly3({{0, 0}, {360, 0}, {360, 2}, {0, 2}, {0, 0}}); EXPECT(poly3.contains({0, 1})); EXPECT(poly3.contains({2 - 360, 1})); EXPECT(poly3.contains({2, 1})); EXPECT(poly3.contains({2 + 360, 1})); Polygon poly4({{-100, 18}, {21, 30}, {150, 50}, {260, 18}, {-100, 18}}); EXPECT(poly4.contains({-10 - 360, 18})); EXPECT(poly4.contains({-10, 18})); EXPECT(poly4.contains({-10 + 360, 18})); Polygon poly5({{-44.2299698513, 44.8732496764}, {-12.2849279262, 75.2545011911}, {72.2148603917, 76.7993105902}, {196.903572422, 71.1350094603}, {304.194105814, 52.8269579527}, {266.886210026, -17.7495991714}, {108.327652927, 34.8499103834}, {-96.2694736324, -17.4340627522}, {-99.8761719143, 7.28288763265}, {-44.2299698513, 44.8732496764}}); for (double lon = -1, lat = 10; lat < 70; lat += 1) { EXPECT(poly5.contains({lon - 360, lat})); EXPECT(poly5.contains({lon, lat})); EXPECT(poly5.contains({lon + 360, lat})); } constexpr double eps = 0.001; constexpr double globe = 360; Polygon poly6({{0 * globe, 4 + eps}, {1 * globe, 2 + eps}, {2 * globe, 0 + eps}, {3 * globe, -2 + eps}, {4 * globe, -4 + eps}, {4 * globe, -4 - eps}, {3 * globe, -2 - eps}, {2 * globe, 0 - eps}, {1 * globe, 2 - eps}, {0 * globe, 4 - eps}, {0 * globe, 4 + eps}}); const std::vector list_lons{-2. * globe, -globe, 0., globe, 2. * globe}; const std::vector list_lats1{4., 2., 0., -2.}; const std::vector list_lats2{5., 3., 1., -1., -3., -5.}; for (double lon : list_lons) { for (double lat : list_lats1) { EXPECT(poly6.contains({lon + 180., lat - 1.})); EXPECT(poly6.contains({lon, lat})); } for (double lat : list_lats2) { EXPECT_NOT(poly6.contains({lon, lat})); EXPECT_NOT(poly6.contains({lon + 180., lat - 1.})); } } // HEALPix-like equator wedge in longitude Polygon poly( {{0, 1}, {0, 90}, {360, 90}, {360, 1}, {361, 0}, {360, -1}, {360, -90}, {0, -90}, {0, -1}, {1, 0}, {0, 1}}); EXPECT(poly.contains({0, 0})); EXPECT(poly.contains({1, 0})); EXPECT(poly.contains({360, 0})); EXPECT(poly.contains({361, 0})); EXPECT(poly.contains({720, 0})); EXPECT(poly.contains({721, 0})); } SECTION("MIR-566: winding number strict check of edges") { Polygon poly({{110, -34}, {90, -62}, {100, -59}, {110, -50}, {132, -40}, {110, -34}}); EXPECT_NOT(poly.contains({90, -40})); EXPECT_NOT(poly.contains({90, -34})); } SECTION("Simple rectangular polygon") { double lonmin = 0; double lonmax = 360; double lonmid = 0.5 * (lonmin + lonmax); double latmax = 80; double latmin = 0; double latmid = 0.5 * (latmin + latmax); Polygon poly({{lonmin, latmax}, {lonmax, latmax}, {lonmax, latmin}, {lonmin, latmin}, {lonmin, latmax}}); EXPECT(poly.contains({lonmin, latmax})); EXPECT(poly.contains({lonmid, latmax})); EXPECT(poly.contains({lonmax, latmax})); EXPECT(poly.contains({lonmax, latmid})); EXPECT(poly.contains({lonmax, latmin})); EXPECT(poly.contains({lonmid, latmin})); EXPECT(poly.contains({lonmin, latmin})); EXPECT(poly.contains({lonmin, latmid})); // Test contains in/outward of edges constexpr auto eps = 0.001; for (size_t i = 0; i <= 100; ++i) { const auto lon = lonmin + static_cast(i) * (lonmax - lonmin) / 100.; EXPECT(poly.contains({lon, latmin + eps})); EXPECT(poly.contains({lon, latmax - eps})); EXPECT_NOT(poly.contains({lon, latmin - eps})); EXPECT_NOT(poly.contains({lon, latmax + eps})); const auto lat = latmin + static_cast(i) * (latmax - latmin) / 100.; EXPECT(poly.contains({lonmin + eps, lat})); EXPECT(poly.contains({lonmax - eps, lat})); EXPECT(poly.contains({lonmin - eps, lat})); EXPECT(poly.contains({lonmax + eps, lat})); } // Test points at non-canonical coordinates // Default behavior throws EXPECT_THROWS_AS(poly.contains({lonmid, 91.}), BadValue); auto A = PointLonLat::make(lonmid + 360., latmid, lonmin); EXPECT(poly.contains({A.lon(), A.lat()})); auto B = PointLonLat::make(lonmid, 180. - latmid, lonmin); EXPECT(poly.contains({B.lon(), B.lat()})); } SECTION("Parallelogram") { Polygon poly({{0, 0}, {1, 1}, {2, 1}, {1, 0}, {0, 0}}); for (const auto& p : poly) { EXPECT(poly.contains(p)); } EXPECT_NOT(poly.contains({0, 1})); EXPECT_NOT(poly.contains({2, 0})); } SECTION("Degenerate polygon") { Polygon poly({{0, 0}, {2, 0}, {2, 0} /*duplicate*/, {0, 2}, {0, 0}}); for (const auto& p : poly) { EXPECT(poly.contains(p)); } for (const auto& p : Polygon::container_type{{2, 2}}) { EXPECT_NOT(poly.contains(p)); } } SECTION("Self-intersecting polygon") { Polygon poly1({{-1, -1}, {1, 1}, {1, -1}, {-1, 1}, {-1, -1}}); EXPECT(poly1.contains({0, 0})); EXPECT(poly1.contains({-1, 0})); EXPECT(poly1.contains({1, 0})); EXPECT_NOT(poly1.contains({0, 1})); EXPECT_NOT(poly1.contains({0, -1})); Polygon poly2({{-1, -1}, {1, -1}, {-1, 1}, {1, 1}, {-1, -1}}); EXPECT(poly2.contains({0, 0})); EXPECT_NOT(poly2.contains({-1, 0})); EXPECT_NOT(poly2.contains({1, 0})); EXPECT(poly2.contains({0, 1})); EXPECT(poly2.contains({0, -1})); Polygon poly3({{-1, 89}, {1, 89}, {0, 90}, {181, 89}, {179, 89}, {0, 90}, {-1, 89}}); EXPECT(poly3.size() == 7); const std::vector list_lons{-720., -360., 0., 360., 720.}; for (const auto& lon : list_lons) { EXPECT(poly3.contains({lon, 89.})); EXPECT(poly3.contains({lon + 180, 89.})); EXPECT_NOT(poly3.contains({lon + 90, 89.})); EXPECT_NOT(poly3.contains({lon + 270, 89.})); } } SECTION("simplify") { Polygon poly{{0., -1.}, {1., 1.}, {-1., 1.}, {0., -1.}}; poly.simplify(); Polygon expected{{-1., 1.}, {0., -1.}, {1., 1.}}; EXPECT(poly == expected); } SECTION("clipping: empty") { Polygon poly; poly.clip(clipper); EXPECT(poly.empty()); } SECTION("clipping: completely covers the clipping polygon") { Polygon poly{{-2, 0}, {0, -2}, {2, 0}, {0, 2}}; poly.clip(clipper); EXPECT(poly == clipper); } SECTION("clipping: exactly aligns with clipping boundary") { auto poly = clipper; EXPECT(poly == clipper); poly.push_back((clipper.back() + clipper.front()) * 0.5); EXPECT(poly != clipper); poly.clip(clipper); EXPECT(poly == clipper); } SECTION("clipping: completely inside the clipping polygon") { Polygon poly{{0, 0}, {0.5, 0.5}, {-0.5, 0.5}}; auto expected = poly; poly.clip(clipper); EXPECT(poly == expected); } SECTION("clipping: vertix/vertices outside the clipping polygon (1)") { Polygon poly{{-1., 0.5}, {-2., 0.}, {-1., -2.}, {0.5, -1.}}; poly.clip(clipper); Polygon expected{{-1., -1.}, {0.5, -1.}, {-1., 0.5}}; EXPECT(poly == expected); EXPECT(is_approximately_equal(poly.area(), 1.125)); } SECTION("clipping: vertix/vertices outside the clipping polygon (2)") { Polygon poly{{0., 0.}, {2., 0.}, {1., 2.}}; poly.clip(clipper); Polygon expected{{0.5, 1.}, {0., 0.}, {1., 0.}, {1., 1.}}; EXPECT(poly == expected); } SECTION("clipping: vertix/vertices outside the clipping polygon (3)") { Polygon poly{{1., 1.5}, {2., 0.5}, {-0.5, -2.}, {-2., 0.}}; poly.clip(clipper); Polygon expected{{-1., 0.5}, {0., 1.}, {1., 1.}, {1., -0.5}, {0.5, -1.}, {-1., -1.}}; EXPECT(poly == expected); EXPECT(is_approximately_equal(poly.area(), 3.625)); EXPECT(is_approximately_equal(poly.area(true), -3.625)); } SECTION("clipping: vertix/vertices outside the clipping polygon (4)") { Polygon poly{{-2, 0}, {0, -2}, {2, 0}, {0, 2}}; poly.clip(clipper); EXPECT(poly == clipper); EXPECT(is_approximately_equal(poly.area(), 4.)); } SECTION("clipping: vertix/vertices outside the clipping polygon (5)") { Polygon poly{{1., -0.5}, {1., 0.5}, {-1., 0.}}; Polygon clipper{{0.5, 0.}, {0., 0.5}, {0., 0.}}; poly.clip(clipper); Polygon expected{{0., 0.}, {0.5, 0.}, {0.2, 0.3}, {0., 0.25}}; EXPECT(poly == expected); } SECTION("clipping: completely outside the clipping polygon") { Polygon poly{{2, 2}, {3, 3}, {3, 2}}; poly.clip(clipper); EXPECT(poly.empty()); EXPECT(is_approximately_equal(poly.area(), 0.)); } SECTION("clipping: concave polygon where part is clipped but part remains inside.") { Polygon poly{{-2., 2.}, {-2., -2.}, {2., -2.}}; poly.clip(clipper); Polygon expected{{-1, -1}, {1, -1}, {-1, 1}}; EXPECT(poly == expected); EXPECT(is_approximately_equal(poly.area(), 2.)); } SECTION("clipping: polygon intersection") { Polygon poly{{0.000304552, -5.32E-06}, {0, 0.026185917}, {0., 0.}}; Polygon clipper{{0.000304598, 0.008724209}, {-0.000304598, 0.008724209}, {0, -0.008726866}}; poly.clip(clipper); Polygon expected{{0.0001522757595430324, -2.659995799630596e-06}, {0.0002436488701595806, 0.005232302171885906}, {0.0002030449380766552, 0.008724209}, {0, 0.008724209}, {0, 0}}; EXPECT(poly == expected); } } } // namespace eckit::geo::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/memory/0000775000175000017500000000000015161702250015454 5ustar alastairalastaireckit-2.0.7/tests/memory/test_factory.cc0000664000175000017500000001470215161702250020475 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/memory/Builder.h" #include "eckit/memory/Factory.h" #include "eckit/testing/Test.h" #include "eckit/value/Properties.h" namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- class Base { public: Base() = default; Base(const Base&) = default; Base(Base&&) = default; Base& operator=(const Base&) = default; Base& operator=(Base&&) = default; virtual ~Base() = default; virtual std::string foo() const = 0; }; //---------------------------------------------------------------------------------------------------------------------- class Base0 : public Base { public: using builder_t = BuilderT0; static std::string className() { return "eckit_test.Base0"; } }; class A0 : public Base0 { public: static std::string className() { return "eckit_test.A0"; } A0() : s1_("A.0") {} std::string foo() const override { return className() + "." + s1_; } private: std::string s1_; }; class B0 : public Base0 { public: static std::string className() { return "eckit_test.B0"; } B0() : s2_("B.0") {} std::string foo() const override { return className() + "." + s2_; } private: std::string s2_; }; static const ConcreteBuilderT0 builder_A0; static const ConcreteBuilderT0 builder_B0; //---------------------------------------------------------------------------------------------------------------------- class Base1 : public Base { public: using builder_t = BuilderT1; using ARG1 = const Properties&; static std::string className() { return "eckit_test.Base1"; } }; class A1 : public Base1 { public: static std::string className() { return "eckit_test.A1"; } explicit A1(const Properties& p) : s1_(p["mystr"].as() + ".1") {} std::string foo() const override { return className() + "." + s1_; } private: std::string s1_; }; class B1 : public Base1 { public: static std::string className() { return "eckit_test.B1"; } explicit B1(const Properties& p) : s2_(p["mystr"].as() + ".2") {} std::string foo() const override { return className() + "." + s2_; } private: std::string s2_; }; static const ConcreteBuilderT1 builder_A1; static const ConcreteBuilderT1 builder_B1; //---------------------------------------------------------------------------------------------------------------------- class Base2 : public Base { public: using builder_t = BuilderT2; using ARG1 = std::string; using ARG2 = int; static std::string className() { return "eckit_test.Base2"; } }; class A2 : public Base2 { public: static std::string className() { return "eckit_test.A2"; } A2(std::string s, int i) : s1_(s + "." + std::to_string(i)) {} std::string foo() const override { return className() + "." + s1_; } private: std::string s1_; }; class B2 : public Base2 { public: static std::string className() { return "eckit_test.B2"; } B2(std::string s, int i) : s2_(s + "." + std::to_string(i)) {} std::string foo() const override { return className() + "." + s2_; } private: std::string s2_; }; static const ConcreteBuilderT2 builder_A2; static const ConcreteBuilderT2 builder_B2_1; static const ConcreteBuilderT2 builder_B2_2("eckit_test.B2.x"); //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_memory_factory_0") { EXPECT(Factory::build_type() == "eckit_test.Base0"); const auto& factory = Factory::instance(); EXPECT(factory.size() == 2); EXPECT(factory.exists("eckit_test.A0")); EXPECT(factory.exists("eckit_test.B0")); EXPECT(factory.get("eckit_test.A0").name() == "eckit_test.A0"); EXPECT(factory.get("eckit_test.B0").name() == "eckit_test.B0"); std::unique_ptr p1(factory.get("eckit_test.A0").create()); std::unique_ptr p2(factory.get("eckit_test.B0").create()); EXPECT(p1); EXPECT(p2); EXPECT(p1->foo() == "eckit_test.A0.A.0"); EXPECT(p2->foo() == "eckit_test.B0.B.0"); } CASE("test_eckit_memory_factory_1") { EXPECT(Factory::build_type() == "eckit_test.Base1"); const auto& factory = Factory::instance(); EXPECT(factory.size() == 2); EXPECT(factory.exists("eckit_test.A1")); EXPECT(factory.exists("eckit_test.B1")); EXPECT(factory.get("eckit_test.A1").name() == "eckit_test.A1"); EXPECT(factory.get("eckit_test.B1").name() == "eckit_test.B1"); Properties p; p.set("mystr", "lolo"); std::unique_ptr p1(factory.get("eckit_test.A1").create(p)); std::unique_ptr p2(factory.get("eckit_test.B1").create(p)); EXPECT(p1); EXPECT(p2); EXPECT(p1->foo() == "eckit_test.A1.lolo.1"); EXPECT(p2->foo() == "eckit_test.B1.lolo.2"); } CASE("test_eckit_memory_factory_2") { EXPECT(Factory::build_type() == "eckit_test.Base2"); const auto& factory = Factory::instance(); EXPECT(factory.size() == 3); EXPECT(factory.exists("eckit_test.A2")); EXPECT(factory.exists("eckit_test.B2")); EXPECT(factory.get("eckit_test.A2").name() == "eckit_test.A2"); EXPECT(factory.get("eckit_test.B2").name() == "eckit_test.B2"); std::string s("lolo"); std::unique_ptr p1(factory.get("eckit_test.A2").create(s, 42)); std::unique_ptr p2(factory.get("eckit_test.B2").create(s, 42)); std::unique_ptr p3(factory.get("eckit_test.B2.x").create(s, -42)); EXPECT(p1); EXPECT(p2); EXPECT(p3); EXPECT(p1->foo() == "eckit_test.A2.lolo.42"); EXPECT(p2->foo() == "eckit_test.B2.lolo.42"); EXPECT(p3->foo() == "eckit_test.B2.lolo.-42"); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/memory/test_memory_mmap.cc0000664000175000017500000001035615161702250021351 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include #include #include #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Bytes.h" #include "eckit/memory/MMap.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- int get_pagesize() { int pagesize; errno = 0ll; if ((pagesize = sysconf(_SC_PAGE_SIZE)) == -1) { if (errno == 0) { printf("PAGE_SIZE not supported by sysconf implementation\n"); } else { perror("sysconf error."); } exit(EXIT_FAILURE); } return pagesize; } // |--------------------------------|--------------------------------| // ^2G limit ^4Gb // ^--------^ mapped region #define SIZE_2G 2147483648 // 2^31 #define TOTAL_SIZE 2 * SIZE_2G // 4Gb file #define ELEMS_2G (SIZE_2G / sizeof(int)) #define NELEMS 8 * 1024 #define MAP_SIZE (NELEMS * sizeof(int)) #define ELEM_LW (ELEMS_2G - NELEMS / 2) #define ELEM_UP (ELEM_LW + NELEMS) #define SIZE_UP (ELEM_UP * sizeof(int)) #define SIZE_LW (ELEM_LW * sizeof(int)) //---------------------------------------------------------------------------------------------------------------------- CASE("Test memory map") { // compute the 2GB limit and its number of pages int pagesize = get_pagesize(); size_t nbpages_2gb = SIZE_2G / pagesize; EXPECT(nbpages_2gb * pagesize == SIZE_2G); std::cout << Bytes(SIZE_2G) << " are " << nbpages_2gb << " memory pages" << std::endl; // mmapped array of int's int* map; // make temporary file name int fd; char filename[] = "mmaped.XXXXXX"; if ((fd = ::mkstemp(filename)) == -1) { perror("cannot create temporary file"), exit(EXIT_FAILURE); } // file is opened in mkstemp, so this is not needed // if( ( fd = ::open(filename, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600) ) == -1 ) // perror("Error opening file"), exit(EXIT_FAILURE); // stretch the file size to the size of the (mmapped) array of ints if ((long long)::lseek(fd, TOTAL_SIZE - 1, SEEK_SET) < 0) { perror("Error calling lseek() to 'stretch' the file"), exit(EXIT_FAILURE); } // write something to the end of the file to actually resize it correctly if (::write(fd, "", 1) != 1) { close(fd), perror("Error writing last byte of the file"), exit(EXIT_FAILURE); } // map the file std::cout << "mapping " << Bytes(MAP_SIZE) << " from " << Bytes(SIZE_LW) << " to " << Bytes(SIZE_UP) << std::endl; if ((map = (int*)MMap::mmap(nullptr, MAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, SIZE_LW)) == MAP_FAILED) { close(fd), perror("Error mmapping the file"), exit(EXIT_FAILURE); } // write to file as if it were memory for (int i = 0; i < NELEMS; ++i) { map[i] = 2 * i; } // free the mmapped memory if (MMap::munmap(map, MAP_SIZE) == -1) { perror("Error un-mmapping the file"), exit(EXIT_FAILURE); } // close the file if (close(fd) == -1) { perror("Error closing file descriptot"), exit(EXIT_FAILURE); } // test that the contents of the file are correct // remove the file -- its large! if (::unlink(filename) == -1) { perror("Error removing the file"), exit(EXIT_FAILURE); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/memory/test_counted.cc0000664000175000017500000000474215161702250020472 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/memory/Counted.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- struct TestCounted : public Counted { TestCounted(int val, bool& destroyed) : i(val), destroyed_(destroyed) { destroyed_ = false; } ~TestCounted() override { destroyed_ = true; } int i; bool& destroyed_; }; //---------------------------------------------------------------------------------------------------------------------- CASE("test_count_starts_at_zero") { bool destroyed = false; auto* obj = new TestCounted(42, destroyed); EXPECT(obj->count() == 0); // Must attach before detach to trigger cleanup obj->attach(); obj->detach(); EXPECT(destroyed); } CASE("test_attach_detach") { bool destroyed = false; auto* obj = new TestCounted(10, destroyed); obj->attach(); EXPECT(obj->count() == 1); EXPECT(!destroyed); obj->detach(); // count reaches 0, triggers delete EXPECT(destroyed); } CASE("test_multiple_attach") { bool destroyed = false; auto* obj = new TestCounted(20, destroyed); obj->attach(); EXPECT(obj->count() == 1); obj->attach(); EXPECT(obj->count() == 2); obj->attach(); EXPECT(obj->count() == 3); obj->detach(); EXPECT(obj->count() == 2); EXPECT(!destroyed); obj->detach(); EXPECT(obj->count() == 1); EXPECT(!destroyed); obj->detach(); // count reaches 0, triggers delete EXPECT(destroyed); } CASE("test_data_access") { bool destroyed = false; auto* obj = new TestCounted(99, destroyed); obj->attach(); EXPECT(obj->i == 99); obj->i = 42; EXPECT(obj->i == 42); obj->detach(); EXPECT(destroyed); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/memory/CMakeLists.txt0000664000175000017500000000104415161702250020213 0ustar alastairalastairoption( ENABLE_ECKIT-351 "Control if ECKIT-351 tests should be enabled" ON ) ecbuild_add_test( TARGET eckit_test_memory_counted SOURCES test_counted.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_memory_factory SOURCES test_factory.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_memory_mmap SOURCES test_memory_mmap.cc LIBS eckit CONDITION ENABLE_ECKIT-351 ) eckit-2.0.7/tests/utils/0000775000175000017500000000000015161702250015304 5ustar alastairalastaireckit-2.0.7/tests/utils/test_safe_casts.cc0000664000175000017500000001172215161702250020770 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/testing/Test.h" #include "eckit/utils/SafeCasts.h" #include #include using eckit::into_signed; using eckit::into_unsigned; CASE("Can convert positive signed to unsigned") { EXPECT_EQUAL(into_unsigned(int8_t{5}), uint8_t{5}); EXPECT_EQUAL(into_unsigned(int16_t{5}), uint16_t{5}); EXPECT_EQUAL(into_unsigned(int32_t{5}), uint32_t{5}); EXPECT_EQUAL(into_unsigned(int64_t{5}), uint64_t{5}); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); } CASE("Throws 'BadCast' on negative signed to unsigned") { EXPECT_THROWS_AS((void)into_unsigned(int8_t{-5}), eckit::BadCast); EXPECT_THROWS_AS((void)into_unsigned(int16_t{-5}), eckit::BadCast); EXPECT_THROWS_AS((void)into_unsigned(int32_t{-5}), eckit::BadCast); EXPECT_THROWS_AS((void)into_unsigned(int64_t{-5}), eckit::BadCast); EXPECT_THROWS_AS((void)into_unsigned(static_cast(-5)), eckit::BadCast); EXPECT_THROWS_AS((void)into_unsigned(static_cast(-5)), eckit::BadCast); EXPECT_THROWS_AS((void)into_unsigned(static_cast(-5)), eckit::BadCast); EXPECT_THROWS_AS((void)into_unsigned(static_cast(-5)), eckit::BadCast); EXPECT_THROWS_AS((void)into_unsigned(static_cast(-5)), eckit::BadCast); } CASE("No-op if used as unsigned to unsigned cast") { EXPECT_EQUAL(into_unsigned(uint8_t{5}), 5); EXPECT_EQUAL(into_unsigned(uint16_t{5}), 5); EXPECT_EQUAL(into_unsigned(uint32_t{5}), 5); EXPECT_EQUAL(into_unsigned(uint64_t{5}), 5); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); EXPECT_EQUAL(into_unsigned(static_cast(5)), 5); } CASE("Can convert unsigned to signed for unsigned values smaller that 2^(bits-1)-1") { EXPECT_EQUAL(into_signed(uint8_t{5}), int8_t{5}); EXPECT_EQUAL(into_signed(uint16_t{5}), int16_t{5}); EXPECT_EQUAL(into_signed(uint32_t{5}), int32_t{5}); EXPECT_EQUAL(into_signed(uint64_t{5}), int64_t{5}); EXPECT_EQUAL(into_signed(static_cast(5)), 5); EXPECT_EQUAL(into_signed(static_cast(5)), 5); EXPECT_EQUAL(into_signed(static_cast(5)), 5); EXPECT_EQUAL(into_signed(static_cast(5)), 5); EXPECT_EQUAL(into_signed(static_cast(5)), 5); } CASE("Throws 'BadCast' on unsigned to signed conversion if value cannot be represented") { EXPECT_THROWS_AS((void)into_signed(uint8_t{std::numeric_limits::max()}), eckit::BadCast); EXPECT_THROWS_AS((void)into_signed(uint16_t{std::numeric_limits::max()}), eckit::BadCast); EXPECT_THROWS_AS((void)into_signed(uint32_t{std::numeric_limits::max()}), eckit::BadCast); EXPECT_THROWS_AS((void)into_signed(uint64_t{std::numeric_limits::max()}), eckit::BadCast); EXPECT_THROWS_AS((void)into_signed(static_cast(std::numeric_limits::max())), eckit::BadCast); EXPECT_THROWS_AS((void)into_signed(static_cast(std::numeric_limits::max())), eckit::BadCast); EXPECT_THROWS_AS((void)into_signed(static_cast(std::numeric_limits::max())), eckit::BadCast); EXPECT_THROWS_AS((void)into_signed(static_cast(std::numeric_limits::max())), eckit::BadCast); EXPECT_THROWS_AS((void)into_signed(static_cast(std::numeric_limits::max())), eckit::BadCast); } CASE("No-op if used as signed to signed cast") { EXPECT_EQUAL(into_signed(int8_t{-5}), -5); EXPECT_EQUAL(into_signed(int16_t{-5}), -5); EXPECT_EQUAL(into_signed(int32_t{-5}), -5); EXPECT_EQUAL(into_signed(int64_t{-5}), -5); EXPECT_EQUAL(into_signed(static_cast(-5)), -5); EXPECT_EQUAL(into_signed(static_cast(-5)), -5); EXPECT_EQUAL(into_signed(static_cast(-5)), -5); EXPECT_EQUAL(into_signed(static_cast(-5)), -5); EXPECT_EQUAL(into_signed(static_cast(-5)), -5); } int main(int argc, char* argv[]) { return eckit::testing::run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_byteswap.cc0000664000175000017500000002434615161702250020521 0ustar alastairalastair/* * (C) Copyright 2020- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/log/Log.h" #include "eckit/types/Types.h" #include "eckit/utils/ByteSwap.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Low-level roundtrip 16 bits") { uint16_t v = 123; uint16_t s = eckit::bitswap16(v); // std::cout << s << std::endl; EXPECT(s == 31488); uint16_t r = eckit::bitswap16(s); EXPECT(r == v); } CASE("Low-level roundtrip 32 bits") { uint32_t v = 1234; uint32_t s = eckit::bitswap32(v); std::cout << s << std::endl; EXPECT(s == 3523477504); uint32_t r = eckit::bitswap32(s); EXPECT(r == v); } CASE("Low-level roundtrip 64 bits") { uint64_t v = 12345; uint64_t s = eckit::bitswap64(v); std::cout << s << std::endl; EXPECT(s == 4120793659044003840); uint64_t r = eckit::bitswap64(s); EXPECT(r == v); } //---------------------------------------------------------------------------------------------------------------------- #if eckit_LITTLE_ENDIAN // use htons and htonl as correctness tests, and test against specifc bit patterns CASE("Check correctness 16 bit swap") { uint16_t v = 511; std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT_EQUAL(bits_to_str(v), "0000000111111111"); uint16_t r = htons(v); std::cout << r << " = " << bits_to_str(r) << std::endl; EXPECT_EQUAL(bits_to_str(r), "1111111100000001"); eckit::byteswap(v); std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT_EQUAL(bits_to_str(v), "1111111100000001"); } CASE("Check correctness 16 bit swap") { // special case of symmetric swap // std::string bitstr("1010101010101010"); // 43690 // std::bitset<16> s(bitstr); // unsigned long v = s.to_ulong(); // std::cout << v << " = " << bits_to_str(v) << std::endl; uint16_t v = 43690; std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT_EQUAL(bits_to_str(v), "1010101010101010"); uint16_t r = htons(v); std::cout << r << " = " << bits_to_str(r) << std::endl; EXPECT_EQUAL(bits_to_str(r), "1010101010101010"); byteswap(v); std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT_EQUAL(bits_to_str(v), "1010101010101010"); } CASE("Check correctness 16 bit swap") { // special case of anti-symmetric swap // std::string bitstr("0101010110101010"); // 21930 // std::bitset<16> s(bitstr); // unsigned long v = s.to_ulong(); // std::cout << v << " = " << bits_to_str(v) << std::endl; uint16_t v = 21930; std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT_EQUAL(bits_to_str(v), "0101010110101010"); uint16_t r = htons(v); std::cout << r << " = " << bits_to_str(r) << std::endl; EXPECT_EQUAL(bits_to_str(r), "1010101001010101"); eckit::byteswap(v); std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT_EQUAL(bits_to_str(v), "1010101001010101"); } CASE("Check correctness 32 bit swap") { uint32_t v = 2212345511; std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT_EQUAL(bits_to_str(v), "10000011110111011011011010100111"); uint32_t r = htonl(v); std::cout << r << " = " << bits_to_str(r) << std::endl; EXPECT_EQUAL(bits_to_str(r), "10100111101101101101110110000011"); eckit::byteswap(v); std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT_EQUAL(bits_to_str(v), "10100111101101101101110110000011"); } CASE("Check correctness 64 bit swap") { union u64 { uint64_t v; struct { uint32_t hi; uint32_t lo; } s; }; u64 v; v.v = 891067242212345511; std::cout << v.v << " = " << bits_to_str(v.v) << std::endl; EXPECT_EQUAL(bits_to_str(v.v), "0000110001011101101101001110111110001011000011011100011010100111"); uint32_t r_hi = htonl(v.s.hi); uint32_t r_lo = htonl(v.s.lo); std::cout << bits_to_str(r_hi) + bits_to_str(r_lo) << std::endl; EXPECT_EQUAL(bits_to_str(r_hi) + bits_to_str(r_lo), "1010011111000110000011011000101111101111101101000101110100001100"); #if defined(htonll) uint64_t sr = htonll(v.v); std::cout << bits_to_str(sr) << std::endl; EXPECT_EQUAL(bits_to_str(sr), "1010011111000110000011011000101111101111101101000101110100001100"); #endif eckit::byteswap(v.v); std::cout << bits_to_str(v.v) << std::endl; EXPECT_EQUAL(bits_to_str(v.v), "1010011111000110000011011000101111101111101101000101110100001100"); } //---------------------------------------------------------------------------------------------------------------------- template void test_scalar_value(T v, std::string bitstr, std::string swapped) { T ref = v; std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT(bits_to_str(v) == bitstr); eckit::byteswap(v); std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT(bits_to_str(v) == swapped); eckit::byteswap(v); std::cout << v << " = " << bits_to_str(v) << std::endl; EXPECT(bits_to_str(v) == bitstr); EXPECT(ref == v); } CASE("ByteSwap short") { SECTION("short 1234") { test_scalar_value(1234, "0000010011010010", "1101001000000100"); } SECTION("short 3145") { test_scalar_value(3145, "0000110001001001", "0100100100001100"); } SECTION("short 32767") { test_scalar_value(32767, "0111111111111111", "1111111101111111"); } } CASE("ByteSwap int") { SECTION("int 2212345511") { test_scalar_value(212345511, "00001100101010000010001010100111", "10100111001000101010100000001100"); } SECTION("int 1024*1024") { test_scalar_value(1024 * 1024, "00000000000100000000000000000000", "00000000000000000001000000000000"); } } CASE("ByteSwap long") { SECTION("long 7") { test_scalar_value(7, "0000000000000000000000000000000000000000000000000000000000000111", "0000011100000000000000000000000000000000000000000000000000000000"); } SECTION("long 870633123454325131") { test_scalar_value(870633123454325131, "0000110000010101000111000011011010000101001011110001000110001011", "1000101100010001001011111000010100110110000111000001010100001100"); } } CASE("ByteSwap double") { SECTION("double exact integer 7.0") { test_scalar_value(7.0, "0100000000011100000000000000000000000000000000000000000000000000", "0000000000000000000000000000000000000000000000000001110001000000"); } SECTION("double PI") { test_scalar_value(M_PI, "0100000000001001001000011111101101010100010001000010110100011000", "0001100000101101010001000101010011111011001000010000100101000000"); } SECTION("double max") { test_scalar_value(std::numeric_limits::max(), "0111111111101111111111111111111111111111111111111111111111111111", "1111111111111111111111111111111111111111111111111110111101111111"); } } #endif // eckit_LITTLE_ENDIAN //---------------------------------------------------------------------------------------------------------------------- template std::vector build_reference() { std::vector ref(1024); for (size_t i = 0; i < ref.size(); ++i) { ref[i] = T(i); } return ref; } template std::ostream& print_bits(std::ostream& out, const std::vector& v) { for (size_t i = 0; i < v.size(); ++i) { out << bits_to_str(v[i]) << "\n"; } out << std::endl; return out; } template void test_roundtrip_vector() { // std::cout << "sizeof(T)*8 " << sizeof(T)*8 << std::endl; const auto ref = build_reference(); // std::cout << "reference" << std::endl; print_bits(std::cout, ref); auto v = ref; // std::cout << "v initial" << std::endl; print_bits(std::cout, v); // First roundtrip with array API eckit::byteswap(v); // std::cout << "v swapped" << std::endl; print_bits(std::cout, v); // std::cout << "v[1/2] " << v[v.size() / 2] << std::endl; EXPECT(v != ref); const auto swap = v; // save swaped state eckit::byteswap(v.data(), v.size()); // std::cout << "v back" << std::endl; print_bits(std::cout, v); EXPECT(v == ref); // Second roundtrip with scalar API for (auto& x : v) { eckit::byteswap(x); } // std::cout << "v elem swapped" << std::endl; print_bits(std::cout, v); EXPECT(v != ref); EXPECT(v == swap); eckit::byteswap(v); // std::cout << "v elem swapped back" << std::endl; print_bits(std::cout, v); EXPECT(v == ref); } CASE("test_roundtrip_vector") { SECTION("short") { test_roundtrip_vector(); } SECTION("unsigned short") { test_roundtrip_vector(); } SECTION("int") { test_roundtrip_vector(); } SECTION("unsigned int") { test_roundtrip_vector(); } SECTION("long") { test_roundtrip_vector(); } SECTION("unsigned long") { test_roundtrip_vector(); } SECTION("long long") { test_roundtrip_vector(); } SECTION("unsigned long long") { test_roundtrip_vector(); } SECTION("float") { test_roundtrip_vector(); } SECTION("double") { test_roundtrip_vector(); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_hashing.cc0000664000175000017500000002066615161702250020305 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/utils/Hash.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit { namespace test { //---------------------------------------------------------------------------------------------------------------------- std::vector tests = {"", "a", "abc", "message digest", "abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", "12345678901234567890123456789012345678901234567890123456789012345678901234567890", "The quick brown fox jumps over the lazy cog"}; std::map> results = { {"nOnE", { "", //"" "", //"a" "", //"abc" "", //"message digest" "", //"abcdefghijklmnopqrstuvwxyz" "", //"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "", //"12345678901234567890123456789012345678901234567890123456789012345678901234567890" "", //"The quick brown fox jumps over the lazy cog" "", //"The quick brown fox jumps over the lazy cog" x 2 }}, {"md4", { "31d6cfe0d16ae931b73c59d7e0c089c0", //"" "bde52cb31de33e46245e05fbdbd6fb24", //"a" "a448017aaf21d8525fc10ae87aa6729d", //"abc" "d9130a8164549fe818874806e1c7014b", //"message digest" "d79e1c308aa5bbcdeea8ed63df412da9", //"abcdefghijklmnopqrstuvwxyz" "043f8582f241db351ce627e153e7f0e4", //"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "e33b4ddc9c38f2199c3e7b164fcc0536", //"12345678901234567890123456789012345678901234567890123456789012345678901234567890" "b86e130ce7028da59e672d56ad0113df", //"The quick brown fox jumps over the lazy cog" "16b52abf32d2796d53c6202a877a69a0", //"The quick brown fox jumps over the lazy cog" x 2 }}, {"MD4", { "31d6cfe0d16ae931b73c59d7e0c089c0", //"" "bde52cb31de33e46245e05fbdbd6fb24", //"a" "a448017aaf21d8525fc10ae87aa6729d", //"abc" "d9130a8164549fe818874806e1c7014b", //"message digest" "d79e1c308aa5bbcdeea8ed63df412da9", //"abcdefghijklmnopqrstuvwxyz" "043f8582f241db351ce627e153e7f0e4", //"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "e33b4ddc9c38f2199c3e7b164fcc0536", //"12345678901234567890123456789012345678901234567890123456789012345678901234567890" "b86e130ce7028da59e672d56ad0113df", //"The quick brown fox jumps over the lazy cog" "16b52abf32d2796d53c6202a877a69a0", //"The quick brown fox jumps over the lazy cog" x 2 }}, {"MD5", { "d41d8cd98f00b204e9800998ecf8427e", //"" "0cc175b9c0f1b6a831c399e269772661", //"a" "900150983cd24fb0d6963f7d28e17f72", //"abc" "f96b697d7cb7938d525a2f31aaf161d0", //"message digest" "c3fcd3d76192e4007dfb496cca67e13b", //"abcdefghijklmnopqrstuvwxyz" "d174ab98d277d9f5a5611c2c9f419d9f", //"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "57edf4a22be3c955ac49da2e2107b67a", //"12345678901234567890123456789012345678901234567890123456789012345678901234567890" "1055d3e698d289f2af8663725127bd4b", //"The quick brown fox jumps over the lazy cog" "f6c40218ba016d723a32a43e3c96a929", //"The quick brown fox jumps over the lazy cog" x 2 }}, {"SHA1", { "da39a3ee5e6b4b0d3255bfef95601890afd80709", //"" "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", //"a" "a9993e364706816aba3e25717850c26c9cd0d89d", //"abc" "c12252ceda8be8994d5fa0290a47231c1d16aae3", //"message digest" "32d10c7b8cf96570ca04ce37f2a19d84240d3a89", //"abcdefghijklmnopqrstuvwxyz" "761c457bf73b14d27e9e9265c46f4b4dda11f940", //"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "50abf5706a150990a08b2c5ea40fa0e585554732", //"12345678901234567890123456789012345678901234567890123456789012345678901234567890" "de9f2c7fd25e1b3afad3e85a0bd17d9b100db4b3", //"The quick brown fox jumps over the lazy cog" "6f020371ced46a2989a2161a113f34ccd35e0729", //"The quick brown fox jumps over the lazy cog" x 2 }}, {"xxh64", { "ef46db3751d8e999", //"" "d24ec4f1a98c6e5b", //"a" "44bc2cf5ad770999", //"abc" "066ed728fceeb3be", //"message digest" "cfe1f278fa89835c", //"abcdefghijklmnopqrstuvwxyz" "aaa46907d3047814", //"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "e04a477f19ee145d", //"12345678901234567890123456789012345678901234567890123456789012345678901234567890" "2dcf47703493b6ca", //"The quick brown fox jumps over the lazy cog" "e32a7da747f1bd6e", //"The quick brown fox jumps over the lazy cog" x 2 }}, {"xxhash", // Deprecated alias of xxh64 { "ef46db3751d8e999", //"" "d24ec4f1a98c6e5b", //"a" "44bc2cf5ad770999", //"abc" "066ed728fceeb3be", //"message digest" "cfe1f278fa89835c", //"abcdefghijklmnopqrstuvwxyz" "aaa46907d3047814", //"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "e04a477f19ee145d", //"12345678901234567890123456789012345678901234567890123456789012345678901234567890" "2dcf47703493b6ca", //"The quick brown fox jumps over the lazy cog" "e32a7da747f1bd6e", //"The quick brown fox jumps over the lazy cog" x 2 }}, }; void testHash(Hash& hash, std::string hashName) { auto hashResults = results.find(hashName); if (hashResults == results.end()) return; for (int i = 0; i < tests.size(); i++) { std::string testResults = hashResults->second[i]; // if (testResults.empty()) // break; hash.reset(); hash.add(tests[i].c_str(), tests[i].size()); EXPECT(testResults == hash.digest()); // constructor-based initialization std::unique_ptr hashInitialized{HashFactory::instance().build(hashName, tests[i])}; EXPECT(testResults == hashInitialized->digest()); // initialize with dummy data and test reset hashInitialized.reset(HashFactory::instance().build(hashName, "dummy data")); hashInitialized->reset(); hashInitialized->add(tests[i].c_str(), tests[i].size()); EXPECT(testResults == hashInitialized->digest()); } // test incremental add { std::string testResults = hashResults->second[tests.size()]; hash.reset(); hash.add(tests[tests.size() - 1].c_str(), tests[tests.size() - 1].size()); hash.add(tests[tests.size() - 1].c_str(), tests[tests.size() - 1].size()); EXPECT(testResults == hash.digest()); // test two adds (as before) is same as a combination of one compute std::string combined; combined += tests[tests.size() - 1]; combined += tests[tests.size() - 1]; EXPECT(hash.compute(combined.c_str(), combined.size()) == testResults); } } CASE("Hashing") { std::unique_ptr hash; SECTION("Default Hashing") { EXPECT_NO_THROW(hash.reset(HashFactory::instance().build())); testHash(*hash, "MD5"); } SECTION("Not Existing Hashing") { EXPECT_THROWS(HashFactory::instance().build("dummy name")); } for (auto const& element : results) { if (HashFactory::instance().has(element.first)) { SECTION(element.first + " Hashing") { EXPECT_NO_THROW(hash.reset(HashFactory::instance().build(element.first))); testHash(*hash, element.first); } } } } //---------------------------------------------------------------------------------------------------------------------- } // end namespace test } // end namespace eckit int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_regex.cc0000664000175000017500000000214015161702250017761 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/Regex.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Escape special regex chars in string") { EXPECT(Regex::escape(".^$*+-?()[]{}\\|") == "\\.\\^\\$\\*\\+\\-\\?\\(\\)\\[\\]\\{\\}\\\\\\|"); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_rendezvoushash.cc0000664000175000017500000001406615161702250021731 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/types/Types.h" #include "eckit/utils/RendezvousHash.h" #include "eckit/utils/Translator.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("test_eckit_utils_rendezvous_hash_constructor") { Log::info() << Here() << std::endl; std::vector nodes = {"node01", "node02", "node03", "node04"}; eckit::RendezvousHash rendezvous(nodes, &RendezvousHash::md5); std::map dict; dict["class"] = "od"; dict["stream"] = "oper"; dict["type"] = "fc"; dict["level"] = "1"; std::vector indices; std::vector node_order; rendezvous.hashOrder(dict, indices); rendezvous.hashOrder(dict, node_order); EXPECT(indices.size() == 4); std::sort(indices.begin(), indices.end()); EXPECT(indices == std::vector({0, 1, 2, 3})); EXPECT(nodes.size() == 4); for (const std::string& node : nodes) { EXPECT(std::find(node_order.begin(), node_order.end(), node) != node_order.end()); } } CASE("test_eckit_utils_rendezvous_hash_distribution") { Log::info() << Here() << std::endl; eckit::Translator toStr; std::vector params{"2t", "2d", "z", "u", "v", "130.128", "138.128"}; std::vector nodes = {"node01", "node02", "node03", "node04", "node05", "node06", "node07"}; eckit::RendezvousHash rendezvous(nodes); std::map dict{{"class", "od"}, {"stream", "oper"}, {"type", "fc"}}; std::map counts; std::vector ordered_nodes; for (std::vector::iterator param = params.begin(); param != params.end(); ++param) { dict["param"] = *param; for (size_t step = 0; step < 240; ++step) { dict["step"] = toStr(step); for (size_t level = 0; level < 10; ++level) { dict["level"] = toStr(level); rendezvous.hashOrder(dict, ordered_nodes); EXPECT(ordered_nodes.size() == 7); counts[ordered_nodes[0]]++; } } } // Test that we have << roughly >> similar counts on all the nodes. (Very stochastic). for (const auto& it : counts) { EXPECT(2320 < it.second && 2450 > it.second); } } CASE("test_eckit_utils_rendezvous_hash_empty_dict") { Log::info() << Here() << std::endl; std::map dict; eckit::RendezvousHash rendezvous; rendezvous.addNode("node01"); rendezvous.addNode("node02"); std::vector indices; EXPECT_NO_THROW(rendezvous.hashOrder(dict, indices)); /* don't throw on empty dictionary */ } CASE("test_eckit_utils_rendezvous_hash_throws_empty_node_list") { Log::info() << Here() << std::endl; std::map dict; eckit::RendezvousHash rendezvous(&RendezvousHash::md5); dict["class"] = "od"; dict["stream"] = "oper"; dict["type"] = "fc"; std::vector indices; EXPECT_THROWS_AS(rendezvous.hashOrder(dict, indices), eckit::BadParameter); /* throw on empty node list */ rendezvous.addNode("node01"); rendezvous.addNode("node02"); EXPECT_NO_THROW(rendezvous.hashOrder(dict, indices)); } CASE("test_eckit_utils_rendezvous_hash_add_node") { Log::info() << Here() << std::endl; eckit::Translator toStr; std::vector params{"2t", "2d", "z", "w"}; std::map dict{{"class", "od"}, {"stream", "oper"}, {"type", "fc"}}; eckit::RendezvousHash rendezvous{std::vector({"node01", "node02"})}; const size_t nsteps = 25; const size_t nlevels = 100; std::map counts; std::vector ordered_nodes; for (std::vector::iterator param = params.begin(); param != params.end(); ++param) { dict["param"] = *param; for (size_t step = 0; step < nsteps; ++step) { dict["step"] = toStr(step); for (size_t level = 0; level < nlevels; ++level) { dict["level"] = toStr(level); rendezvous.hashOrder(dict, ordered_nodes); EXPECT(ordered_nodes.size() == 2); counts[ordered_nodes[0]]++; } } } // Test that we have << roughly >> similar counts on all the nodes. (Very stochastic). for (auto kv : counts) { EXPECT(4950 < kv.second && 5050 > kv.second); } std::map counts2; rendezvous.addNode("node03"); rendezvous.addNode("node04"); for (std::vector::iterator param = params.begin(); param != params.end(); ++param) { dict["param"] = *param; for (size_t step = 0; step < nsteps; ++step) { dict["step"] = toStr(step); for (size_t level = 0; level < nlevels; ++level) { dict["level"] = toStr(level); rendezvous.hashOrder(dict, ordered_nodes); EXPECT(ordered_nodes.size() == 4); counts2[ordered_nodes[0]]++; } } } // Test that we have << roughly >> similar counts on all the nodes. (Very stochastic). for (auto kv : counts2) { EXPECT(2430 < kv.second && 2551 > kv.second); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_translator.cc0000664000175000017500000001211115161702250021037 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Translator.h" #include "eckit/filesystem/PathName.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Translate strings to integers") { Translator toSize; EXPECT(toSize("0") == size_t(0)); EXPECT(toSize("1") == size_t(1)); EXPECT(toSize("100") == size_t(100)); EXPECT(toSize("1KB") == size_t(1024)); EXPECT(toSize("1MB") == size_t(1024 * 1024)); EXPECT(toSize("1GB") == size_t(1024 * 1024 * 1024)); // EXPECT( toSize("1TB") == size_t(1024*1024*1024*1024)); // EXPECT( toSize("1PB") == size_t(1024*1024*1024*1024*1024)); // EXPECT( toSize("1EB") == size_t(1024*1024*1024*1024*1024*1024)); EXPECT(toSize("1KiB") == size_t(1024)); EXPECT(toSize("1MiB") == size_t(1024 * 1024)); EXPECT(toSize("1GiB") == size_t(1024 * 1024 * 1024)); std::cout << toSize("1GiB") << std::endl; std::cout << toSize("1TiB") << std::endl; EXPECT(toSize("1TiB") == size_t(1LL << 40)); EXPECT(toSize("1PiB") == size_t(1LL << 50)); EXPECT(toSize("1EiB") == size_t(1LL << 60)); // Translator toInt; // Translator toUInt; // Translator toLong; // Translator toULong; } CASE("Translate strings to double") { double half = 0.5; double zero = 0.0; double hund = 100.0; double ten = 10.; Translator t; EXPECT(half == t("0.5")); EXPECT(hund == t("1e2")); EXPECT(hund == t("1E2")); EXPECT(hund == t("1.e2")); EXPECT(hund == t("1.E2")); EXPECT(hund == t("1.0e2")); EXPECT(hund == t("1.0E2")); EXPECT(hund == t("1e002")); EXPECT(hund == t("1E002")); EXPECT(hund == t("1.0e002")); EXPECT(hund == t("1.0E002")); EXPECT(zero == t("0")); EXPECT(zero == t("0.")); EXPECT(zero == t("0.0")); EXPECT(zero == t("0.00")); EXPECT(zero == t("0.0e0")); EXPECT(zero == t("0.0e-0")); EXPECT(zero == t("0.0e+0")); EXPECT(zero == t("+0")); EXPECT(zero == t("+0.")); EXPECT(zero == t("+0.0")); EXPECT(zero == t("+0.00")); EXPECT(zero == t("+0.0e0")); EXPECT(zero == t("+0.0e-0")); EXPECT(zero == t("+0.0e+0")); EXPECT(zero == t("-0")); EXPECT(zero == t("-0.")); EXPECT(zero == t("-0.0")); EXPECT(zero == t("-0.00")); EXPECT(zero == t("-0.0e0")); EXPECT(zero == t("-0.0e-0")); EXPECT(zero == t("-0.0e+0")); /// weird cases that actually pass EXPECT_NO_THROW(t("inf")); // inf is acceptable, case insensitive EXPECT_NO_THROW(t("INF")); // INF is acceptable, case insensitive EXPECT_NO_THROW(t("infinity")); // infinity is acceptable, case insensitive EXPECT_NO_THROW(t("INFINITY")); // INFINITY is acceptable, case insensitive EXPECT_NO_THROW(t("nan")); // nan is acceptable, case insensitive EXPECT_NO_THROW(t("NAN")); // NAN is acceptable, case insensitive EXPECT(zero == t("0x0")); // hexadecimal is acceptable EXPECT(ten == t("0xA")); // hexadecimal is acceptable /// these should fail ... EXPECT_THROWS_AS(t(""), BadParameter); // empty string EXPECT_THROWS_AS(t("0.5 "), BadParameter); // no spaces accepted -- we are being extra strict EXPECT_THROWS_AS(t(" 0.5"), BadParameter); // no spaces accepted -- we are being extra strict EXPECT_THROWS_AS(t(" 0.5 "), BadParameter); // no spaces accepted -- we are being extra strict EXPECT_THROWS_AS(t("1e+10000"), BadParameter); // overflow, max ~ 1.79769e+308 EXPECT_THROWS_AS(t("1e-10000"), BadParameter); // underflow, min ~ 2.22507e-308 EXPECT_THROWS_AS(t("0.5a"), BadParameter); EXPECT_THROWS_AS(t("foobar"), BadParameter); EXPECT_THROWS_AS(t("foo555bar"), BadParameter); } CASE("Translate signed char as a number") { Translator t; EXPECT(t((signed char)127) == std::string("127")); EXPECT(t((signed char)-127) == std::string("-127")); EXPECT(t((signed char)128) == std::string("-128")); } CASE("Translate through construction (including implicit conversion)") { Translator t; EXPECT(t("123") == 123); EXPECT(t(eckit::PathName("123")) == 123); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_rle.cc0000664000175000017500000001034715161702250017441 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/log/Timer.h" #include "eckit/utils/RLE.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- void test(const std::vector& in, EncodingClock::duration timeout = std::chrono::milliseconds(2), long depth = -1, long expectedSize = -1) { std::vector out; std::vector outTimeout; int maxLoop = 5000; long long n; long long t; { eckit::Timer timer; n = RLEencode2(in.begin(), in.end(), std::back_inserter(out), maxLoop); LOG_DEBUG_LIB(LibEcKit) << "no_timeout - time elapsed: " << timer.elapsed() << " - output size: " << out.size() << std::endl; } { eckit::Timer timer; t = RLEencode2(in.begin(), in.end(), std::back_inserter(outTimeout), maxLoop, timeout, depth); LOG_DEBUG_LIB(LibEcKit) << "timeout " << std::chrono::duration_cast(timeout).count() << "ms - time elapsed: " << timer.elapsed() << " - output size: " << outTimeout.size() << std::endl; } if (expectedSize > 0) { EXPECT_EQUAL(expectedSize, n); EXPECT_EQUAL(expectedSize, t); EXPECT_EQUAL(out.size(), outTimeout.size()); for (size_t i = 0; i < out.size(); i++) { EXPECT_EQUAL(out[i], outTimeout[i]); } } std::vector decoded; RLEdecode2(out.begin(), out.end(), std::back_inserter(decoded)); EXPECT_EQUAL(in.size(), decoded.size()); for (size_t i = 0; i < in.size(); i++) { EXPECT_EQUAL(in[i], decoded[i]); } std::vector decodedTimeout; RLEdecode2(outTimeout.begin(), outTimeout.end(), std::back_inserter(decodedTimeout)); EXPECT_EQUAL(in.size(), decodedTimeout.size()); for (size_t i = 0; i < in.size(); i++) { EXPECT_EQUAL(in[i], decodedTimeout[i]); } } CASE("single") { std::vector in = {4}; test(in, std::chrono::seconds(1), 10, 1); } CASE("identical") { size_t size = 1000; std::vector in; in.reserve(size); for (size_t i = 0; i < size; i++) { in.push_back(4); } test(in, std::chrono::seconds(1), 10, 2); /// we expect [-1000,4] } CASE("interleaved") { size_t size = 1000; std::vector in; in.reserve(size); for (size_t i = 0; i < size; i++) { in.push_back(i % 2); } test(in, std::chrono::seconds(1), 10, 4); /// we expect [-500,-2,0,1] } CASE("pattern") { size_t size = 100000; std::vector in; in.reserve(size); for (size_t i = 0; i < size; i++) { in.push_back(std::ceil(std::sqrt(i))); } test(in, std::chrono::milliseconds(50), 1000); } void randTest(int limit, int size) { std::random_device rd; // a seed source for the random number engine std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd() std::uniform_int_distribution<> distrib(1, limit); std::vector in; in.reserve(size); for (size_t i = 0; i < size; i++) { in.push_back(distrib(gen)); } test(in, std::chrono::seconds(2), 1000); } CASE("randMaxInt") { randTest(std::numeric_limits::max(), 10000000); } CASE("rand100") { randTest(100, 10000000); } CASE("rand10") { randTest(10, 1000000); } CASE("rand2") { randTest(2, 1000000); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_compressor.cc0000664000175000017500000001033315161702250021046 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/io/Buffer.h" #include "eckit/utils/Compressor.h" #include "eckit/utils/MD5.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { static std::string msg("THE QUICK BROWN FOX JUMPED OVER THE LAZY DOG'S BACK 1234567890"); static std::vector compressions{"none", "snappy", "lz4", "bzip2", "aec"}; //---------------------------------------------------------------------------------------------------------------------- void EXPECT_compress_uncompress_1(Compressor& c, const Buffer& in, size_t ulen) { // Buffers are not allocated. The compress/uncompress will do so as required Buffer compressed; Buffer uncompressed; size_t clen = c.compress(in, ulen, compressed); c.uncompress(compressed, clen, uncompressed, ulen); EXPECT(std::memcmp(uncompressed, in, ulen) == 0); } void EXPECT_compress_uncompress_2(Compressor& c, const Buffer& in, size_t ulen) { // Buffers are pre-allocated. This may allow implementation dependent optimizations Buffer compressed(size_t(1.2 * ulen)); Buffer uncompressed(size_t(1.2 * ulen)); size_t clen = c.compress(in, ulen, compressed); c.uncompress(compressed, clen, uncompressed, ulen); EXPECT(std::memcmp(uncompressed, in, ulen) == 0); } void EXPECT_reproducible_compression(Compressor& c, size_t times) { std::vector reproduce_lengths; std::vector reproduce_checksum; for (size_t i = 0; i < times + 1; ++i) { Buffer uncompressed(msg.data(), msg.size()); Buffer compressed; size_t compressed_size = c.compress(uncompressed, uncompressed.size(), compressed); reproduce_lengths.emplace_back(compressed_size); reproduce_checksum.emplace_back(eckit::MD5(compressed, compressed_size)); } const auto& ref_length = reproduce_lengths.front(); bool reproducible_length{true}; for (auto& length : reproduce_lengths) { if (length != ref_length) { reproducible_length = false; } } EXPECT(reproducible_length); const auto& ref_checksum = reproduce_checksum.front(); bool reproducible_checksum{true}; for (auto& checksum : reproduce_checksum) { if (checksum != ref_checksum) { reproducible_checksum = false; } } EXPECT(reproducible_checksum); } CASE("Builders") { std::unique_ptr c; SECTION("CASE No Compression - case insensitive") { EXPECT_NO_THROW(c.reset(CompressorFactory::instance().build("nOnE"))); } SECTION("Not Existing Compression") { EXPECT_THROWS(c.reset(CompressorFactory::instance().build("dummy name"))); } } CASE("Compression") { Buffer in(2 * msg.size()); // oversized on purpose in.copy(msg.c_str(), msg.size()); const size_t len = msg.size(); // valid size std::unique_ptr c; SECTION("CASE Default Compression") { EXPECT_NO_THROW(c.reset(CompressorFactory::instance().build())); EXPECT_compress_uncompress_1(*c, in, len); EXPECT_compress_uncompress_2(*c, in, len); } for (const auto& compression : CompressorFactory::instance().keys()) { SECTION("CASE " + compression) { if (CompressorFactory::instance().has(compression)) { EXPECT_NO_THROW(c.reset(CompressorFactory::instance().build(compression))); EXPECT_compress_uncompress_1(*c, in, len); EXPECT_compress_uncompress_2(*c, in, len); EXPECT_reproducible_compression(*c, 10); } } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_literals.cc0000664000175000017500000000160415161702250020472 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/utils/Literals.h" #include "eckit/testing/Test.h" using namespace eckit::literals; using namespace eckit::testing; namespace eckit::test { CASE("size literals") { EXPECT_EQUAL(1_KiB, 1ULL << 10); EXPECT_EQUAL(1_MiB, 1ULL << 20); EXPECT_EQUAL(1_GiB, 1ULL << 30); EXPECT_EQUAL(1_TiB, 1ULL << 40); EXPECT_EQUAL(1_PiB, 1ULL << 50); EXPECT_EQUAL(1_EiB, 1ULL << 60); } } // namespace eckit::test int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/hash-performance.cc0000664000175000017500000000516615161702250021045 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/io/Buffer.h" #include "eckit/log/Bytes.h" #include "eckit/log/Seconds.h" #include "eckit/log/Timer.h" #include "eckit/utils/Hash.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit { namespace test { //---------------------------------------------------------------------------------------------------------------------- template void timeAdd(Hash& hash, eckit::Buffer& buffer, eckit::Timer& timer) { timer.start(); for (int i = 0; i < N; ++i) { hash.reset(); for (size_t j = 0; j < M; ++j) { hash.add(buffer, buffer.size()); } hash.digest(); } timer.stop(); std::cout << " - " << N << " x " << M << " x add(" << Bytes(buffer.size()) << ") rate " << Bytes(N * M * buffer.size(), timer) << std::endl; } template void timeCompute(Hash& hash, eckit::Buffer& buffer, eckit::Timer& timer) { timer.start(); for (int i = 0; i < N; ++i) { std::string s = hash.compute(buffer, buffer.size()); assert(!s.empty()); } timer.stop(); std::cout << " - " << N << " x compute(" << Bytes(buffer.size()) << ") rate " << Bytes(N * buffer.size(), timer) << std::endl; } //---------------------------------------------------------------------------------------------------------------------- CASE("Test hash performance") { eckit::Buffer buffer(4 * 1024 * 1024); eckit::Buffer buffer2(64 * 1024 * 1024); eckit::Timer timer; std::vector hashes{"xxh64", "MD4", "MD5", "SHA1"}; for (auto& name : hashes) { if (eckit::HashFactory::instance().has(name)) { std::cout << name << std::endl; std::unique_ptr hash(eckit::HashFactory::instance().build(name)); timeAdd<20, 1>(*hash, buffer, timer); timeCompute<5>(*hash, buffer2, timer); } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace test } // namespace eckit int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_string_tools.cc0000664000175000017500000001100715161702250021377 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/runtime/Tool.h" #include "eckit/types/Types.h" #include "eckit/utils/StringTools.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Convert to lower") { std::string istr = "Foo Bar"; std::string ostr = StringTools::lower(istr); EXPECT(ostr == "foo bar"); } CASE("Convert to upper") { std::string istr = "Foo Bar"; std::string ostr = StringTools::upper(istr); EXPECT(ostr == "FOO BAR"); } CASE("substitute") { StringDict m; m["class"] = "od"; m["stream"] = "oper"; string in("{class}:none:{stream}"); string out = StringTools::substitute(in, m); EXPECT(out == "od:none:oper"); } CASE("listVariables") { string in("{class}:none:{stream}"); StringList out = StringTools::listVariables(in); EXPECT(out.size() == 2); EXPECT(out[0] == "class"); EXPECT(out[1] == "stream"); } CASE("startsWith") { string in("_lolo_test"); string s1("_lolo"); string s2("lolo"); string s3("_lolo_test_bigger"); EXPECT(!StringTools::startsWith(in, "")); EXPECT(StringTools::startsWith(in, in)); EXPECT(StringTools::startsWith(in, s1)); EXPECT(StringTools::startsWith(in, "_")); EXPECT(!StringTools::startsWith(in, s2)); EXPECT(!StringTools::startsWith(in, s3)); } CASE("endsWith") { string in("_lolo_test"); EXPECT(!StringTools::endsWith(in, "")); EXPECT(StringTools::endsWith(in, in)); EXPECT(StringTools::endsWith(in, "t")); string s1("_test"); EXPECT(StringTools::endsWith(in, s1)); string s2("lolo"); EXPECT(!StringTools::endsWith(in, s2)); string s3("_lolo_test333"); EXPECT(!StringTools::endsWith(in, s3)); } CASE("trim") { string t1(" lolo_test "); EXPECT(StringTools::trim(t1) == string("lolo_test")); string t2(" lolo_test"); EXPECT(StringTools::trim(t2) == string("lolo_test")); string t3("lolo_test "); EXPECT(StringTools::trim(t3) == string("lolo_test")); string t4(""); EXPECT(StringTools::trim(t4) == string("")); string t5("nothing_here"); EXPECT(StringTools::trim(t5) == string("nothing_here")); string t6("XXXXXXusefullXXXXX"); EXPECT(StringTools::trim(t6, "X") == string("usefull")); string t7("0000010"); EXPECT(StringTools::trim(t7, "0") == string("1")); } CASE("front_trim") { string t1(" lolo_test "); EXPECT(StringTools::front_trim(t1) == string("lolo_test ")); string t2(" lolo_test"); EXPECT(StringTools::front_trim(t2) == string("lolo_test")); string t3("lolo_test "); EXPECT(StringTools::front_trim(t3) == string("lolo_test ")); string t4(""); EXPECT(StringTools::front_trim(t4) == string("")); string t5("nothing_here"); EXPECT(StringTools::front_trim(t5) == string("nothing_here")); string t6("XXXXXXusefullXXXXX"); EXPECT(StringTools::front_trim(t6, "X") == string("usefullXXXXX")); string t7("0000010"); EXPECT(StringTools::front_trim(t7, "0") == string("10")); } CASE("back_trim") { string t1(" lolo_test "); Log::info() << StringTools::back_trim(t1) << std::endl; EXPECT(StringTools::back_trim(t1) == string(" lolo_test")); string t2(" lolo_test"); EXPECT(StringTools::back_trim(t2) == string(" lolo_test")); string t3("lolo_test "); EXPECT(StringTools::back_trim(t3) == string("lolo_test")); string t4(""); EXPECT(StringTools::back_trim(t4) == string("")); string t5("nothing_here"); EXPECT(StringTools::back_trim(t5) == string("nothing_here")); string t6("XXXXXXusefullXXXXX"); EXPECT(StringTools::back_trim(t6, "X") == string("XXXXXXusefull")); string t7("0000010"); EXPECT(StringTools::back_trim(t7, "0") == string("000001")); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/compression-performance.cc0000664000175000017500000001110715161702250022453 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include #include #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/log/Bytes.h" #include "eckit/log/Seconds.h" #include "eckit/log/Timer.h" #include "eckit/utils/Compressor.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit { namespace test { //---------------------------------------------------------------------------------------------------------------------- template size_t timeCompress(const Compressor& compressor, eckit::Buffer& inBuffer, eckit::Buffer& outBuffer, eckit::Timer& timer) { timer.start(); size_t out; for (int i = 0; i < N; ++i) { out = compressor.compress(inBuffer, inBuffer.size(), outBuffer); } timer.stop(); std::cout << " - compress() rate " << Bytes(N * inBuffer.size(), timer) << " -- factor " << std::fixed << std::setprecision(2) << ((100.0 * out) / inBuffer.size()) << "% -- "; std::cout << std::fixed << std::setprecision(1) << (out / 1024.0) << "/" << (inBuffer.size() / 1024.0) << " KB" << std::endl; return out; } template void timeDecompress(const Compressor& compressor, eckit::Buffer& inBuffer, size_t inlen, eckit::Buffer& outBuffer, size_t outlen, eckit::Timer& timer) { timer.start(); for (int i = 0; i < N; ++i) { compressor.uncompress(inBuffer, inlen, outBuffer, outlen); } timer.stop(); std::cout << " - decompress() rate " << Bytes(N * outlen, timer) << std::endl; } struct BinaryData { eckit::Buffer in; std::string description; BinaryData(const eckit::PathName& path, const std::string& desc) : in(path.size()), description(desc) { std::unique_ptr dh(path.fileHandle()); dh->openForRead(); dh->read(in, in.size()); dh->close(); } }; //---------------------------------------------------------------------------------------------------------------------- CASE("Test compression performance") { eckit::Timer timer; std::vector data; data.emplace_back("2t_sfc.grib", "GRIB t2 surface layer"); data.emplace_back("2t_sfc_regrid.grib", "GRIB t2 surface layer re-gridded"); data.emplace_back("vo-d_6ml.grib", "GRIB vo/d layers (10-15 spherical harmonics)"); data.emplace_back("u-v_6ml.grib", "GRIB u/v layers (10-15)"); data.emplace_back("q_6ml_regrid.grib", "GRIB q 6 layers (10-15) re-gridded"); std::vector compressors{"none", "lz4", "snappy", "aec", "bzip2"}; constexpr int N = 5; // Number of iterations to use for each case auto test_case = [&](const Compressor& compressor, BinaryData& data) { // Allocations here to prevent them from being timed in timings during first iteration // Allocation with more than required can prevent internal buffers and memcopies, depending on implementation auto ulen = data.in.size(); size_t oversize = size_t(1.2 * ulen); // used in AEC Buffer compressed{oversize}; Buffer uncompressed{oversize}; // touch memory compressed.zero(); uncompressed.zero(); // Compress size_t clen = timeCompress(compressor, data.in, compressed, timer); // Decompress timeDecompress(compressor, compressed, clen, uncompressed, ulen, timer); }; for (const auto& name : compressors) { if (eckit::CompressorFactory::instance().has(name)) { std::cout << name << std::endl; std::unique_ptr compressor(eckit::CompressorFactory::instance().build(name)); for (auto& d : data) { std::cout << " " << d.description << std::endl; test_case(*compressor, d); } } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace test } // namespace eckit int main(int argc, char* argv[]) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/test_semanticversion.cc0000664000175000017500000000734015161702250022067 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/runtime/Tool.h" #include "eckit/types/SemanticVersion.h" #include "eckit/types/Types.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Empty versions") { SemanticVersion v; EXPECT(std::string(v) == "0.0.0"); SemanticVersion v0("0.0.0"); SemanticVersion v1("1.0.0"); SemanticVersion v2("0.1.0"); SemanticVersion v3("0.0.1"); EXPECT(v == v0); EXPECT(v < v1); EXPECT(v < v2); EXPECT(v < v3); } CASE("Construct with integers") { EXPECT(SemanticVersion() == SemanticVersion(0, 0, 0)); EXPECT(SemanticVersion("0.0.0") == SemanticVersion(0, 0, 0)); EXPECT(SemanticVersion("27.125.22") == SemanticVersion(27, 125, 22)); EXPECT(SemanticVersion("2.1.3") != SemanticVersion(27, 125, 22)); } CASE("Failed construction") { EXPECT_THROWS_AS(SemanticVersion("1"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("2.3"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("1.2.3.4"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("1.2-4"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("1.2.4-r1"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("1.-2.4"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("1.2.456abc"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("1.aaa.56"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("v1.2.4"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("x.y.z"), eckit::BadValue); EXPECT_THROWS_AS(SemanticVersion("1.2.999999999999999999999999999999"), eckit::OutOfRange); } CASE("Version comparison") { std::ostringstream ss; ss << SemanticVersion("27.125.22"); EXPECT(ss.str() == "27.125.22"); } CASE("Version comparison") { SemanticVersion v1("1.0.0"); SemanticVersion v2("0.1.0"); SemanticVersion v3("0.0.1"); EXPECT(std::string(v1) == "1.0.0"); EXPECT(std::string(v2) == "0.1.0"); EXPECT(std::string(v3) == "0.0.1"); EXPECT(v1 > v2); EXPECT(v1 > v3); EXPECT(v2 < v1); EXPECT(v2 > v3); EXPECT(v3 < v1); EXPECT(v3 < v2); } CASE("Multidigit versions") { SemanticVersion v("27.125.22"); EXPECT(std::string(v) == "27.125.22"); EXPECT(v == SemanticVersion("27.125.22")); EXPECT(v != SemanticVersion("27.125.33")); EXPECT(v != SemanticVersion("27.25.22")); EXPECT(v != SemanticVersion("7.125.22")); EXPECT(v > SemanticVersion("27.125.11")); EXPECT(v > SemanticVersion("27.1.33")); EXPECT(v > SemanticVersion("2.125.2")); EXPECT(v < SemanticVersion("27.125.33")); EXPECT(v < SemanticVersion("27.777.33")); EXPECT(v < SemanticVersion("42.125.22")); EXPECT(v >= SemanticVersion("27.125.22")); EXPECT(v >= SemanticVersion("27.125.12")); EXPECT(v <= SemanticVersion("27.125.22")); EXPECT(v <= SemanticVersion("27.125.33")); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/utils/CMakeLists.txt0000664000175000017500000000506115161702250020046 0ustar alastairalastair### test data files list( APPEND test_data_files 2t_sfc.grib 2t_sfc_regrid.grib vo-d_6ml.grib u-v_6ml.grib q_6ml_regrid.grib ) if(HAVE_EXTRA_TESTS) ecbuild_get_test_multidata( TARGET get_eckit_test_data NAMES ${test_data_files} NOCHECK ) endif() ecbuild_add_test( TARGET eckit_test_utils_byteswap SOURCES test_byteswap.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_string_tools SOURCES test_string_tools.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_translator SOURCES test_translator.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_tokenizer SOURCES test_tokenizer.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_semanticversion SOURCES test_semanticversion.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_rendezvoushash SOURCES test_rendezvoushash.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_rle SOURCES test_rle.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_compressor SOURCES test_compressor.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_hashing CONDITION eckit_HAVE_SSL INCLUDES "${OPENSSL_INCLUDE_DIR}" SOURCES test_hashing.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_hash_performance CONDITION HAVE_EXTRA_TESTS SOURCES hash-performance.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_utils_compression_performance CONDITION HAVE_EXTRA_TESTS TEST_DEPENDS get_eckit_test_data SOURCES compression-performance.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_regex SOURCES test_regex.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_literals SOURCES test_literals.cc LIBS eckit ) ecbuild_add_test( TARGET eckit_test_safe_casts SOURCES test_safe_casts.cc LIBS eckit ) eckit-2.0.7/tests/utils/test_tokenizer.cc0000664000175000017500000000721415161702250020670 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/runtime/Tool.h" #include "eckit/types/Types.h" #include "eckit/utils/Tokenizer.h" #include "eckit/testing/Test.h" using namespace std; using namespace eckit; using namespace eckit::testing; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- template void test_single() { std::string source(":lolo1:lolo2::lolo3"); Container target; Tokenizer parse(":"); parse(source, target); EXPECT(target.size() == 3); size_t c = 1; for (typename Container::const_iterator i = target.begin(); i != target.end(); ++i, ++c) { ostringstream s; s << "lolo" << c; // std::cout << *i << " ??? " << s.str() << std::endl; EXPECT(*i == s.str()); } } //---------------------------------------------------------------------------------------------------------------------- template void test_multi() { Container target; Tokenizer parse("-:;"); std::string source1("-lolo0-lolo1-lolo2;lolo3:-lolo4-"); parse(source1, target); EXPECT(target.size() == 5); size_t c = 0; for (typename Container::const_iterator i = target.begin(); i != target.end(); ++i, ++c) { ostringstream s; s << "lolo" << c; // std::cout << *i << " ??? " << s.str() << std::endl; EXPECT(*i == s.str()); } std::string source2("-lolo5-lolo6-lolo7;lolo8:lolo9-"); parse(source2, target); EXPECT(target.size() == 10); c = 0; for (typename Container::const_iterator i = target.begin(); i != target.end(); ++i, ++c) { ostringstream s; s << "lolo" << c; // std::cout << *i << " ??? " << s.str() << std::endl; EXPECT(*i == s.str()); } } //---------------------------------------------------------------------------------------------------------------------- void test_keep_empty_list() { StringList target; Tokenizer parse(":", true); std::string source1(":4i:2100:::01:::::."); parse(source1, target); Log::info() << target << std::endl; EXPECT(target.size() == 11); } //---------------------------------------------------------------------------------------------------------------------- void test_keep_empty_set() { StringSet target; Tokenizer parse(":", true); std::string source1(":4i:2100:::01:::::."); parse(source1, target); Log::info() << target << std::endl; EXPECT(target.size() == 5); // 5 uniqe elments } //---------------------------------------------------------------------------------------------------------------------- CASE("Test Tokenizer StringList") { test_single(); test_multi(); test_keep_empty_list(); } CASE("Test Tokenizer StringSet") { test_single(); test_multi(); test_keep_empty_set(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::test //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char** argv) { return run_tests(argc, argv); } eckit-2.0.7/tests/value/0000775000175000017500000000000015161702250015260 5ustar alastairalastaireckit-2.0.7/tests/value/test_value_boolean.cc0000664000175000017500000003365315161702250021453 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" #include "eckit/value/Value.h" #include "test_value_helper.h" using namespace std; using namespace eckit; using namespace eckit::testing; using namespace eckit::test::value_helper; using eckit::types::is_approximately_equal; namespace eckit::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Booleans cast correctly in/out of Value") { Value val_true(true); Value val_false(false); // // Access and conversion of bools // EXPECT(bool(val_true)); EXPECT(!bool(val_false)); EXPECT(val_true.as()); EXPECT(!val_false.as()); // Integer type conversions EXPECT(int(val_true) == 1); EXPECT(int(val_false) == 0); EXPECT(val_true.as() == 1); EXPECT(val_false.as() == 0); // For pretty printing EXPECT(std::string(val_true) == "true"); EXPECT(std::string(val_false) == "false"); EXPECT(val_true.as() == "true"); EXPECT(val_false.as() == "false"); // ValueList is a bit of an odd one --> it just puts the value in a list of one element... ValueList vl_true(val_true.as()); ValueList vl_false = val_false; EXPECT(vl_true.size() == 1); EXPECT(vl_false.size() == 1); EXPECT(vl_true[0].as() == true); EXPECT(vl_false[0].as() == false); // And all the invalid conversions // For some reason, Value(bool) happily converts to double... // FIXME: EXPECT_THROWS_AS(val_false.as(), BadConversion); EXPECT(Value(false).as() == 0.0); // FIXME: Do we want this? EXPECT(Value(true).as() == 1.0); // FIXME: Do we want this? // Length/Offset are just integers, so bool-->Offset conversion works...!!! // FIXME: EXPECT_THROWS_AS(Length(val_false), BadConversion); // FIXME: EXPECT_THROWS_AS(Offset(val_false), BadConversion); EXPECT_THROWS_AS(val_false.as"; } else { s << ""; } } //---------------------------------------------------------------------------------------------------------------------- void Html::Substitute::substitute(std::ostream& s, const std::string& p) { std::map >::iterator i = map_.find(p); if (i == map_.end()) { s << '%' << p << '%'; } else { s << HttpStream::doEncode << (*i).second << HttpStream::dontEncode; } } Html::Substitute::Substitute() {} Html::Substitute::~Substitute() {} std::string& Html::Substitute::operator[](const std::string& p) { return map_[p]; } void Html::Class::print(std::ostream& s) const { std::string p; long len = str_.length(); std::string base = "http://wwwec.ecmwf.int/dhs/classfinder?file="; for (int i = 0; i < len; i++) { char c = str_[i]; if (isalnum(c) || c == '_') { p += c; } else if (p.length()) { s << Link(base + p) << p << Link(); s << c; p = ""; } else { s << c; } } if (p.length()) { s << Link(base + p) << p << Link(); } } void Html::BeginForm::print(std::ostream& s) const { s << "
"; } void Html::EndForm::print(std::ostream& s) const { s << "
"; } void Html::TextField::print(std::ostream& s) const { s << title_ << HttpStream::dontEncode; s << ""; s << HttpStream::doEncode; } void Html::HiddenField::print(std::ostream& s) const { s << ""; } void Html::CheckBox::print(std::ostream& s) const { s << ""; } void Html::Button::print(std::ostream& s) const { s << ""; } void Html::BeginTextArea::print(std::ostream& s) const { s << HttpStream::dontEncode; s << "" << std::endl << HttpStream::doEncode; } //---------------------------------------------------------------------------------------------------------------------- class ImageProvider : public HtmlResource { public: ImageProvider() : HtmlResource(Html::Image::resource()) {} void GET(std::ostream&, Url&); }; static ImageProvider imageProvider; void ImageProvider::GET(std::ostream& out, Url& url) { eckit::PathName path = eckit::Resource("imagePath", "~/html/image"); for (int i = 1; i < url.size(); i++) { path = path + "/" + url[i]; } std::ifstream in(path.localPath()); if (!in) { url.status(HttpError::NOT_FOUND); // Not Found out << path << ": " << Log::syserr << std::endl; } else { (url.headerOut()).type("image/gif"); out << HttpStream::dontEncode; char c; while (in.get(c)) { out << c; } out << HttpStream::doEncode; } } //---------------------------------------------------------------------------------------------------------------------- class HtmlProvider : public HtmlResource { public: HtmlProvider() : HtmlResource("/html") {} virtual ~HtmlProvider() {} void GET(std::ostream&, Url&); }; static HtmlProvider htmlProvider; void HtmlProvider::GET(std::ostream& s, Url& url) { std::string path; for (int i = 1; i < url.size(); i++) { path += "/" + url[i]; } Html::Substitute empty; Html::Include include(path, empty); s << include; } //---------------------------------------------------------------------------------------------------------------------- void Html::BeginTable::print(std::ostream& s) const { s << ""; } void Html::TableTag::print(std::ostream& s) const { s << '<' << tag_; if (align_) { if ((align_ & Center)) { s << " ALIGN=center"; } if ((align_ & Left)) { s << " ALIGN=left"; } if ((align_ & Right)) { s << " ALIGN=right"; } if ((align_ & Top)) { s << " VALIGN=top"; } if ((align_ & Bottom)) { s << " VALIGN=bottom"; } } if (colspan_) { s << " COLSPAN=" << colspan_; } if (rowspan_) { s << " ROWSPAN=" << rowspan_; } s << '>'; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/web/HtmlObject.cc0000664000175000017500000000417415161702250020015 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/web/HtmlObject.h" #include "eckit/web/Url.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- HtmlObject::HtmlObject() {} HtmlObject::~HtmlObject() {} void HtmlObject::HEAD(std::ostream& s, Url& url) { std::ostringstream oss; oss << url.method() << " not supported for " << *this << " (url=" << url << ")"; throw eckit::UserError(oss.str()); } void HtmlObject::GET(std::ostream& s, Url& url) { // std::ostringstream oss; // oss << url.method() << " not supported for " << *this << " (url=" << url << ")"; // throw eckit::UserError(oss.str()); html(s, url); } void HtmlObject::POST(std::ostream& s, Url& url) { // std::ostringstream oss; // oss << url.method() << " not supported for " << *this << " (url=" << url << ")"; // throw eckit::UserError(oss.str()); html(s, url); } void HtmlObject::PUT(std::ostream& s, Url& url) { NOTIMP; } void HtmlObject::DELETE(std::ostream& s, Url& url) { NOTIMP; } void HtmlObject::TRACE(std::ostream& s, Url& url) { NOTIMP; } void HtmlObject::OPTIONS(std::ostream& s, Url& url) { NOTIMP; } void HtmlObject::CONNECT(std::ostream& s, Url& url) { NOTIMP; } void HtmlObject::PATCH(std::ostream& s, Url& url) { NOTIMP; } void HtmlObject::substitute(std::ostream& s, const std::string& p) { s << '%' << p << '%'; } void HtmlObject::print(std::ostream& s) const { s << "No print method defined for this object"; } void HtmlObject::java(JavaAgent&) {} void HtmlObject::html(std::ostream& s, Url& url) {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/web/JavaAgent.h0000664000175000017500000000423615161702250017463 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File JavaAgent.h // Baudouin Raoult - ECMWF Nov 97 #ifndef JavaAgent_H #define JavaAgent_H #include "eckit/serialisation/Streamable.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class JavaAgent : public eckit::Streamable { public: enum { none, user, oper, admin, root }; // -- Contructors JavaAgent(eckit::Stream&); // -- Destructor ~JavaAgent() override; // -- Methods void startObject(const std::string&); void endObject(); eckit::Stream& stream() { return stream_; } virtual void execute(eckit::Stream&, std::istream&, std::ostream&) = 0; virtual int clearance() = 0; // -- Overridden methods void encode(eckit::Stream&) const override; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const eckit::ClassSpec& classSpec() { return classSpec_; } static void serve(eckit::Stream&, std::istream&, std::ostream&); protected: // members eckit::Stream& stream_; std::string user_; protected: // methods virtual void print(std::ostream&) const = 0; private: // members static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; private: // methods friend std::ostream& operator<<(std::ostream& s, const JavaAgent& p) { p.print(s); return s; } friend class JavaUser; }; template <> Streamable* Reanimator::ressucitate(Stream& s) const; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/web/HttpStream.h0000664000175000017500000000223715161702250017715 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_web_HttpStream_H #define eckit_web_HttpStream_H #include "eckit/web/Url.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class HttpBuf; class HttpStream : public std::ostream { public: HttpStream(); ~HttpStream(); void reset(); void write(std::ostream&, Url&, DataHandle&); void print(std::ostream& s) const; static std::ostream& dontEncode(std::ostream&); static std::ostream& doEncode(std::ostream&); private: HttpBuf* buf_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/web/HttpServer.cc0000664000175000017500000000203215161702250020057 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/web/HttpServer.h" #include "eckit/web/HttpService.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- HttpServer::HttpServer(int port, bool visible) : HtmlResource("/"), http_(new HttpService(port, visible)) { http_.start(); } HttpServer::~HttpServer() { http_.stop(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/web/Url.h0000664000175000017500000000675015161702250016370 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult #ifndef eckit_web_Url_h #define eckit_web_Url_h #include #include #include "eckit/net/HttpHeader.h" #include "eckit/value/Value.h" namespace eckit { class Url; //---------------------------------------------------------------------------------------------------------------------- class UrlAccess { Url& url_; std::string s_; public: UrlAccess(Url& url, const std::string& s) : url_(url), s_(s) {} operator std::string(); operator long(); UrlAccess& operator=(const std::string&); UrlAccess& operator=(long); }; //---------------------------------------------------------------------------------------------------------------------- class Url { public: Url(std::istream&); Url(const std::string&); Url(const Url&) = delete; Url& operator=(const Url&) = delete; Url(Url&&) = delete; Url& operator=(Url&&) = delete; ~Url(); UrlAccess operator[](const std::string&); void erase(const std::string&); void set(const std::string&, const std::string&); const std::string& get(const std::string&); const std::string& method() { return method_; } HttpHeader& headerIn(); HttpHeader& headerOut(); const HttpHeader& headerIn() const; const HttpHeader& headerOut() const; std::string str() const; std::string name() const; int size() const; const std::string& operator[](int) const; bool authenticated() { return headerIn().authenticated(); } void authenticate(const std::string& realm = "MARS") { headerOut().authenticate(realm); } void status(int s, const std::string& message = "") { headerOut().status(s, message); } void type(const std::string& type) { headerOut().type(type); } void forward(const std::string& s) { headerOut().forward(s); } void location(const std::string& s) { headerOut().forward(s); } void retryAfter(long s) { headerOut().retryAfter(s); } void dontCache() { headerOut().dontCache(); } void cgiParam(std::ostream&, char sep = ' ') const; eckit::Value json() const; void remaining(const std::vector& remaining) { remaining_ = remaining; } const std::vector& remaining() const { return remaining_; } void streamFrom(DataHandle*, const std::string& type = "application/octet-stream"); DataHandle* streamFrom(); const std::string& streamType() const; protected: // methods void print(std::ostream&) const; private: // methods void parse(const std::string&, bool); void parse(std::istream&); friend std::ostream& operator<<(std::ostream& s, const Url& p) { p.print(s); return s; } private: // members using dict_t = std::map; std::unique_ptr handle_; std::string type_; dict_t dict_; std::vector url_; HttpHeader in_; HttpHeader out_; std::string method_; std::vector remaining_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/web/JavaService.cc0000664000175000017500000000130015161702250020150 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/web/JavaService.h" #include "eckit/web/JavaUser.h" namespace eckit { JavaService::JavaService(int port) : net::NetService(port) {} JavaService::~JavaService() {} net::NetUser* JavaService::newUser(net::TCPSocket& socket) const { return new JavaUser(socket); } } // namespace eckit eckit-2.0.7/src/eckit/web/JavaAgent.cc0000664000175000017500000000412015161702250017611 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/web/JavaAgent.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/os/Password.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- eckit::ClassSpec JavaAgent::classSpec_ = { &Streamable::classSpec(), "JavaAgent", }; Reanimator JavaAgent::reanimator_; void JavaAgent::encode(eckit::Stream& s) const { Streamable::encode(s); } JavaAgent::JavaAgent(eckit::Stream& s) : Streamable(s), stream_(s) {} JavaAgent::~JavaAgent() {} void JavaAgent::startObject(const std::string& clss) { stream_.startObject(); stream_ << clss; } void JavaAgent::endObject() { stream_.endObject(); } void JavaAgent::serve(eckit::Stream& s, std::istream& in, std::ostream& out) { std::unique_ptr a(Reanimator::reanimate(s)); ASSERT(a.get()); Log::info() << *a << std::endl; int agentClearance = a->clearance(); if (agentClearance == JavaAgent::none) { s << int(0); // No need for password } else { s << int(1); // Ask for password; std::string user; std::string password; s >> user; s << Password::salt(user); s >> password; if (!Password::check(user, password)) { s << int(-1); return; } s << int(0); a->user_ = user; } a->execute(s, in, out); } template <> Streamable* eckit::Reanimator::ressucitate(eckit::Stream& s) const { return nullptr; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/web/CgiResource.h0000664000175000017500000000204015161702250020024 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File CgiResource.h // Baudouin Raoult - ECMWF Sep 97 #ifndef CgiResource_H #define CgiResource_H #include "eckit/web/HtmlResource.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class CgiResource : public HtmlResource { public: CgiResource(); ~CgiResource() override; protected: // members std::string name_; protected: // overridden methods void GET(std::ostream&, Url&) override; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/web/CgiResource.cc0000664000175000017500000000326715161702250020176 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/web/CgiResource.h" #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/StdPipe.h" #include "eckit/web/HttpStream.h" #include "eckit/web/Url.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- CgiResource::CgiResource() : HtmlResource("/cgi") {} CgiResource::~CgiResource() {} void CgiResource::GET(std::ostream& s, Url& url) { eckit::PathName cgiScriptsPath = eckit::Resource("cgiScriptsPath", "~/admin/cgi-scripts"); eckit::PathName path(cgiScriptsPath / url.name()); std::ostringstream cmd; std::string mode = url["parameter"]; if (mode == "") { cmd << "env "; url.cgiParam(cmd, ' '); cmd << " " << path; } else { cmd << path << ' '; url.cgiParam(cmd, ' '); } StdPipe pipe(cmd.str(), "r"); AutoCloser closer(pipe); char line[1024]; s << HttpStream::dontEncode; while (fgets(line, sizeof(line), pipe)) { s << line; } s << HttpStream::doEncode; } static CgiResource cgiResourceInstance; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/web/Url.cc0000664000175000017500000001610115161702250016515 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/web/Url.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/log/Log.h" #include "eckit/parser/JSONParser.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" #include "eckit/web/Html.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- inline void header(char c) { #if 0 if (isprint(c)) std::cout << "header: " << c << std::endl; else std::cout << "header: " << hex << int(c) << dec << std::endl; #endif } void Url::parse(const std::string& url, bool param) { int size = url.length(); int index = 0; std::string s, p; while (index < size) { switch (url[index]) { case '?': param = true; if (s != "") { url_.push_back(Html::removeHex(s)); } s = ""; break; case '=': if (param) { p = s; s = ""; } else { s += url[index]; } break; case '&': if (param) { dict_[Html::removeHex(p)] = Html::removeHex(s); s = ""; p = ""; } else { s += url[index]; } break; case '+': // Do we need a multimap? s += url[index]; break; case '/': if (!param) { if (s != "") { url_.push_back(Html::removeHex(s)); } s = ""; } else { s += url[index]; } break; default: s += url[index]; break; } ++index; } if (!param) { if (s != "") { url_.push_back(Html::removeHex(s)); } } else { if (p != "") { dict_[Html::removeHex(p)] = Html::removeHex(s); } } } Url::Url(std::istream& in) { std::string url; in >> method_; in >> url; parse(url, false); char c = 0; while (in.get(c) && c != '\n') { header(c); } parse(in); Log::debug() << "Incomming url-> " << *this << std::endl; } void Url::parse(std::istream& in) { char c = 0; std::map > m; for (;;) { std::string s; while (in.get(c) && c != '\r' && c != ':') { header(c); s += c; } if (c != ':') { header(c); in.get(c); // Skip '\n' header(c); break; } header(c); while (in.get(c) && c == ' ') { header(c); } if (c != '\r') { header(c); std::string r; r += c; while (in.get(c) && c != '\r') { header(c); r += c; } m[s] = r; in.get(c); // Skip '\n' header(c); } } in_ = m; long len = in_.contentLength(); if (len) { static const std::string FormType = "application/x-www-form-urlencoded"; // static const std::string JSONType = "application/json"; // bool ascii = true; Buffer content(len); const std::string& type = in_.type(); char* p = content; for (int i = 0; i < len; i++) { in.get(c); header(c); *p++ = c; } if (type == FormType) { std::string s(static_cast(content), p - static_cast(content)); parse(s, true); } in_.content(content, len); } Log::debug() << *this << std::endl; } Url::Url(const std::string& url) : method_("GET") { parse(url, false); Log::debug() << "Incomming url-> " << *this << std::endl; } Url::~Url() {} std::string Url::name() const { std::string s = ""; for (std::vector::const_iterator j = url_.begin(); j != url_.end(); ++j) { s += "/"; s += *j; } return s; } void Url::print(std::ostream& s) const { for (std::vector::const_iterator j = url_.begin(); j != url_.end(); ++j) { s << "/" << *j; } char c = '?'; dict_t::const_iterator i; for (i = dict_.begin(); i != dict_.end(); ++i) { s << c << (*i).first << '=' << (*i).second; c = '&'; } } void Url::cgiParam(std::ostream& s, char sep) const { char c = ' '; dict_t::const_iterator i; for (i = dict_.begin(); i != dict_.end(); ++i) { s << c << (*i).first << '=' << (*i).second; c = sep; } } UrlAccess Url::operator[](const std::string& s) { return UrlAccess(*this, s); } void Url::set(const std::string& p, const std::string& s) { dict_[p] = s; } const std::string& Url::get(const std::string& s) { return dict_[s]; } void Url::erase(const std::string& s) { dict_.erase(s); } const std::string& Url::operator[](int n) const { return url_[n]; } eckit::Value Url::json() const { std::string p = in_.content(); if (p.empty()) { return toValue(dict_); } std::cout << "================" << std::endl; std::cout << p << std::endl; std::cout << "================" << std::endl; return JSONParser::decodeString(p); } std::string Url::str() const { std::ostringstream s; s << *this; return s.str(); } int Url::size() const { return url_.size(); } HttpHeader& Url::headerIn() { return in_; } HttpHeader& Url::headerOut() { return out_; } void Url::streamFrom(DataHandle* handle, const std::string& type) { handle_.reset(handle); type_ = type; } DataHandle* Url::streamFrom() { return handle_.get(); } const std::string& Url::streamType() const { return type_; } //---------------------------------------------------------------------------------------------------------------------- UrlAccess::operator long() { return Translator()(url_.get(s_)); } UrlAccess::operator std::string() { return url_.get(s_); } UrlAccess& UrlAccess::operator=(long n) { url_.set(s_, Translator()(n)); return *this; } UrlAccess& UrlAccess::operator=(const std::string& s) { url_.set(s_, s); return *this; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/web/HtmlResource.h0000664000175000017500000000207115161702250020232 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_web_HtmlResource_H #define eckit_web_HtmlResource_H #include #include "eckit/web/HttpResource.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// Intermediate Class /// Derive all resources that provide an Html content class HtmlResource : public HttpResource { public: // methods HtmlResource(const std::string&); ~HtmlResource() override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/web/HttpStream.cc0000664000175000017500000001300515161702250020046 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/DataHandle.h" #include "eckit/log/Log.h" #include "eckit/thread/Mutex.h" #include "eckit/web/HttpStream.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class HttpBuf : public std::streambuf { char out_[4096]; virtual int overflow(int c); virtual int sync(); HttpStream& owner_; public: explicit HttpBuf(HttpStream& s); virtual ~HttpBuf(); void reset(); void write(std::ostream&, Url&); void print(std::ostream&) const; private: std::vector buffer_; }; //---------------------------------------------------------------------------------------------------------------------- static int xindex = std::ios::xalloc(); using VC = std::vector; class back_encoder_iterator { VC& container; void push(const char* p) { while (*p != static_cast(0)) { container.push_back(*p++); } } public: using iterator_category = std::output_iterator_tag; using value_type = VC::value_type; using difference_type = std::ptrdiff_t; using pointer = value_type*; using reference = value_type&; explicit back_encoder_iterator(VC& v) : container(v) {} back_encoder_iterator& operator=(value_type); back_encoder_iterator& operator*() { return *this; } back_encoder_iterator& operator++() { return *this; } back_encoder_iterator& operator++(int) { return *this; } }; back_encoder_iterator& back_encoder_iterator::operator=(char c) { switch (c) { case '<': push("<"); break; case '>': push(">"); break; case '&': push("&"); break; case '\n': push("
\n"); break; default: container.push_back(c); break; } return *this; } inline back_encoder_iterator back_encoder(VC& x) { return back_encoder_iterator(x); } //---------------------------------------------------------------------------------------------------------------------- HttpBuf::HttpBuf(HttpStream& s) : owner_(s) { setp(out_, out_ + sizeof(out_)); } HttpBuf::~HttpBuf() { sync(); } void HttpBuf::reset() { ::memset(out_, 0, sizeof(out_)); setp(out_, out_ + sizeof(out_)); buffer_.clear(); } int HttpBuf::sync() { if (owner_.iword(xindex)) { std::copy(pbase(), pptr(), back_encoder(buffer_)); } else { std::copy(pbase(), pptr(), std::back_inserter(buffer_)); } setp(pbase(), epptr()); return 0; } int HttpBuf::overflow(int c) { sync(); if (c == EOF) { return 0; } sputc(c); return 0; } void HttpBuf::write(std::ostream& out, Url& url) { HttpHeader& header = url.headerOut(); // header.length(buffer_.size() + 6 + 7); header.length(buffer_.size()); Log::debug() << "Header: " << std::endl; // Send header out << header; Log::debug() << header; // Send data // out << ""; std::ostream_iterator oi(out); std::copy(buffer_.begin(), buffer_.end(), oi); // out << ""; #if 0 Log::debug() << "Data: " << std::endl; for (std::vector::iterator i = buffer_.begin(); i != buffer_.end(); ++i) if (isprint(*i) || isspace(*i)) Log::debug() << *i; else break; Log::debug() << std::endl; #endif } std::ostream& HttpStream::dontEncode(std::ostream& s) { ASSERT(s.iword(xindex) == 1); // s.rdbuf()->sync(); // << std::flush; s << std::flush; s.iword(xindex) = 0; return s; } std::ostream& HttpStream::doEncode(std::ostream& s) { ASSERT(s.iword(xindex) == 0); // s.rdbuf()->sync(); // << std::flush; s << std::flush; s.iword(xindex) = 1; return s; } void HttpBuf::print(std::ostream& os) const { os << "HttpBuf[buffer=" << buffer_ << "]"; } HttpStream::HttpStream() : std::ostream(new HttpBuf(*this)) { buf_ = (HttpBuf*)rdbuf(); iword(xindex) = 1; // encode } HttpStream::~HttpStream() { delete buf_; } void HttpStream::reset() { buf_->reset(); } void HttpStream::write(std::ostream& s, Url& url, DataHandle& stream) { DataHandle* handle = url.streamFrom(); if (handle) { HttpHeader& header = url.headerOut(); header.length(handle->estimate()); header.type(url.streamType()); AutoClose close(*handle); s << header; s.flush(); if (Log::debug()) { Log::debug() << "Header: " << std::endl; Log::debug() << header; Log::debug() << "Tranfer " << handle->estimate() << " bytes" << std::endl; } handle->saveInto(stream); } else { flush(); buf_->write(s, url); } } void HttpStream::print(std::ostream& s) const { buf_->pubsync(); buf_->print(s); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/web/JavaServer.h0000664000175000017500000000214215161702250017665 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File JavaServer.h // Baudouin Raoult - ECMWF Jun 96 #ifndef JavaServer_H #define JavaServer_H #include "eckit/thread/ThreadControler.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class JavaServer { public: // -- Contructors JavaServer(int port); // -- Destructor ~JavaServer(); private: // No copy allowed JavaServer(const JavaServer&); JavaServer& operator=(const JavaServer&); // -- Members eckit::ThreadControler java_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/web/CMakeLists.txt0000664000175000017500000000141015161702250020201 0ustar alastairalastairlist( APPEND eckit_web_srcs AgentResource.cc AgentResource.h CgiResource.cc CgiResource.h Configure.cc FileResource.cc FileResource.h FtpRequest.h Html.cc Html.h HtmlObject.cc HtmlObject.h HtmlResource.cc HtmlResource.h HttpResource.cc HttpResource.h HttpStream.cc HttpStream.h HttpServer.cc HttpServer.h HttpService.cc HttpService.h HttpUser.h JSONResource.cc JSONResource.h JavaAgent.cc JavaAgent.h JavaResource.cc JavaResource.h JavaServer.cc JavaServer.h JavaService.cc JavaService.h JavaUser.cc JavaUser.h Url.cc Url.h) ecbuild_add_library( TARGET eckit_web TYPE SHARED INSTALL_HEADERS LISTED SOURCES ${eckit_web_srcs} PERSISTENT HtmlObject.h HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/web PUBLIC_LIBS eckit ) eckit-2.0.7/src/eckit/web/JavaResource.h0000664000175000017500000000357615161702250020222 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File JavaResource.h // Baudouin Raoult - ECMWF Sep 97 #ifndef JavaResource_H #define JavaResource_H #include "eckit/web/HtmlResource.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class JavaResource : public HtmlResource { public: // -- Exceptions // None // -- Contructors JavaResource(); // -- Destructor ~JavaResource() override; // -- Convertors // None // -- Operators // None // -- Methods // None // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None std::string name_; // -- Methods // void print(std::ostream&) const; // -- Overridden methods void GET(std::ostream&, Url&) override; // -- Class members // None // -- Class methods private: // No copy allowed JavaResource(const JavaResource&); JavaResource& operator=(const JavaResource&); // -- Members // -- Methods // None // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends // friend std::ostream& operator<<(std::ostream& s,const JavaResource& p) // { p.print(s); return s; } }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/web/HtmlObject.h0000664000175000017500000000327115161702250017654 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File HtmlObject.h // Baudouin Raoult - ECMWF Oct 96 #ifndef HtmlObject_H #define HtmlObject_H #include "eckit/persist/Bless.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Url; class JavaAgent; class Bless; class HtmlObject { public: HtmlObject(); #include "eckit/web/HtmlObject.b" virtual ~HtmlObject(); virtual void java(JavaAgent&); virtual void substitute(std::ostream&, const std::string&); virtual void HEAD(std::ostream&, Url&); virtual void GET(std::ostream&, Url&); virtual void POST(std::ostream&, Url&); virtual void PUT(std::ostream&, Url&); virtual void DELETE(std::ostream&, Url&); virtual void TRACE(std::ostream&, Url&); virtual void OPTIONS(std::ostream&, Url&); virtual void CONNECT(std::ostream&, Url&); virtual void PATCH(std::ostream&, Url&); protected: virtual void html(std::ostream&, eckit::Url&); virtual void print(std::ostream&) const; private: friend std::ostream& operator<<(std::ostream& s, const HtmlObject& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/web/Configure.cc0000664000175000017500000000462415161702250017703 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @note This code is here to avoid linking it in if you don't need it #include "eckit/config/Configurable.h" #include "eckit/config/Resource.h" #include "eckit/config/ResourceMgr.h" #include "eckit/log/Log.h" #include "eckit/web/Html.h" #include "eckit/web/HtmlResource.h" #include "eckit/web/Url.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- void Configurable::htmlAllResources(std::ostream& s, Url& url) { callAll(&Configurable::htmlResources, s, url); } void Configurable::htmlResources(std::ostream& s, Url& url) { for (Set::const_iterator i = resources_.begin(); i != resources_.end(); ++i) { (*i)->GET(s, url); } } void ResourceBase::GET(std::ostream& s, Url& url) { std::string n = name(); std::string u = url["name"]; init(); if (n == u) { std::string v = url[n]; Log::info() << "New value for " << n << ": " << v << std::endl; ResourceMgr::instance().set(n, v); Configurable::reconfigureAll(); } dump(s); s << Html::BeginForm(); s << Html::TextField(n, getValue(), n + ": "); s << Html::SubmitButton(); s << Html::ResetButton(); s << Html::HiddenField("name", n); s << Html::EndForm(); s << Html::Line(); } //---------------------------------------------------------------------------------------------------------------------- class ConfigResource : public HtmlResource { virtual bool restricted() { return true; } virtual void GET(std::ostream&, Url&); public: ConfigResource() : HtmlResource("/config") {} }; void ConfigResource::GET(std::ostream& s, Url& url) { Configurable::htmlAllResources(s, url); } static ConfigResource ostoreResource; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/eckit_version.h.in0000664000175000017500000000101215161702250020304 0ustar alastairalastair#ifndef eckit_version_h #define eckit_version_h #define eckit_VERSION_STR "@eckit_VERSION_STR@" #define eckit_VERSION "@eckit_VERSION@" #define eckit_VERSION_MAJOR @eckit_VERSION_MAJOR@ #define eckit_VERSION_MINOR @eckit_VERSION_MINOR@ #define eckit_VERSION_PATCH @eckit_VERSION_PATCH@ #ifdef __cplusplus extern "C" { #endif const char * eckit_version(); unsigned int eckit_version_int(); const char * eckit_version_str(); const char * eckit_git_sha1(); #ifdef __cplusplus } #endif #endif // eckit_version_h eckit-2.0.7/src/eckit/log/0000775000175000017500000000000015161702250015451 5ustar alastairalastaireckit-2.0.7/src/eckit/log/Channel.h0000664000175000017500000000450615161702250017177 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Channel.h /// @author Tiago Quintino #ifndef eckit_log_Channel_h #define eckit_log_Channel_h #include namespace eckit { class ChannelBuffer; class LogTarget; using channel_callback_t = void (*)(void* data, const char* msg); //---------------------------------------------------------------------------------------------------------------------- /// Output channel that is an std::ostream but more functional class Channel : public std::ostream { public: // methods Channel(LogTarget* = nullptr); Channel(const Channel&) = delete; Channel& operator=(const Channel&) = delete; Channel(Channel&&) = delete; Channel& operator=(Channel&&) = delete; ~Channel() override; bool operator!() const; operator bool() const; void indent(const char* prefix = ""); void unindent(); void setStream(std::ostream& out); void addStream(std::ostream& out); void setFile(const std::string& path); void addFile(const std::string& path); void setCallback(channel_callback_t cb, void* data = nullptr); void addCallback(channel_callback_t cb, void* data = nullptr); void setTarget(LogTarget*); void addTarget(LogTarget*); void reset(); private: // members friend std::ostream& operator<<(std::ostream& os, const Channel& c) { c.print(os); return os; } void print(std::ostream& s) const; ChannelBuffer* buffer_; friend class Log; }; //---------------------------------------------------------------------------------------------------------------------- class AutoIndent { Channel& channel_; public: AutoIndent(Channel& channel, const char* prefix = "") : channel_(channel) { channel_.indent(prefix); } ~AutoIndent() { channel_.unindent(); } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/UserChannel.h0000664000175000017500000000330215161702250020027 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file UserChannel.h /// @author Tiago Quintino #ifndef eckit_log_UserChannel_h #define eckit_log_UserChannel_h namespace eckit { //----------------------------------------------------------------------------- class UserMsg { public: virtual ~UserMsg() {} virtual void infoMsg(const std::string&) = 0; virtual void warningMsg(const std::string&) = 0; virtual void errorMsg(const std::string&) = 0; virtual void notifyClient(const std::string&) = 0; }; //----------------------------------------------------------------------------- class UserBuffer; class UserChannel : public std::ostream { public: // types enum MsgType { NONE, INFO, ERROR, WARN }; public: // methods /// Constructor UserChannel(); UserChannel(const UserChannel&) = delete; UserChannel& operator=(const UserChannel&) = delete; UserChannel(UserChannel&&) = delete; UserChannel& operator=(UserChannel&&) = delete; /// Destructor ~UserChannel(); /// type for next message void msgType(MsgType t); void userMsg(UserMsg*); UserMsg* userMsg() const; protected: UserBuffer* buffer_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Statistics.h0000664000175000017500000000711415161702250017757 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Statistics.h /// @author Baudouin Raoult /// @date April 2016 #ifndef eckit_Statistics_H #define eckit_Statistics_H #include #include "eckit/log/Timer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Stream; struct Timing { double elapsed_; double cpu_; size_t updates_; Timing() : elapsed_(0), cpu_(0), updates_(0) {} Timing(double elapsed, double cpu, size_t updates) : elapsed_(elapsed), cpu_(cpu), updates_(updates) {} Timing(Timer& timer) : elapsed_(timer.elapsed()), cpu_(timer.elapsed_cpu()), updates_(1) {} Timing& operator+=(const Timing&); Timing& operator-=(const Timing&); Timing operator-(const Timing&) const; Timing& operator/=(size_t); }; Stream& operator>>(Stream&, Timing&); Stream& operator<<(Stream&, const Timing&); std::ostream& operator<<(std::ostream&, const Timing&); //---------------------------------------------------------------------------------------------------------------------- class Statistics { public: static void reportUnit(std::ostream& out, const char* title, const char* unit, double value, const char* indent = "", bool always = false); static void reportRate(std::ostream& out, const char* title, unsigned long long value, const char* indent = "", bool always = false); static void reportRate(std::ostream& out, const std::string& title, size_t bytes, double elapsed, const char* indent = "", bool always = false); static void reportCount(std::ostream& out, const char* title, size_t value, const char* indent = "", bool always = false); static void reportBytes(std::ostream& out, const char* title, unsigned long long value, const char* indent = "", bool always = false); static void reportBytesStats(std::ostream& out, const std::string& title, size_t count, size_t bytes, size_t sumsquared, const char* indent = "", bool always = false); static void reportTime(std::ostream& out, const char* title, const Timing& value, const char* indent = "", bool always = false); static void reportTime(std::ostream& out, const char* title, double value, const char* indent = "", bool always = false); static void reportTimeStats(std::ostream& out, const std::string& title, size_t count, double sum_times, double sum_times_squared, const char* indent = "", bool always = false); static Timer& timer() { return timer_; } private: static Timer timer_; }; //---------------------------------------------------------------------------------------------------------------------- class AutoTiming { Timing& timing_; Timing start_; public: AutoTiming(Timing& timing) : timing_(timing), start_(Statistics::timer()) {} ~AutoTiming() { timing_ += Timing(Statistics::timer()) - start_; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/CodeLocation.h0000664000175000017500000000351415161702250020170 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino #ifndef eckit_log_CodeLocation_h #define eckit_log_CodeLocation_h #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class CodeLocation { public: // methods friend std::ostream& operator<<(std::ostream& s, const CodeLocation& loc) { loc.print(s); return s; } /// Empty contructor CodeLocation() : line_(0), file_{nullptr}, func_{nullptr} {} /// Full Contructor CodeLocation(const char* file, int line, const char* func) : line_(line), file_(file), func_(func) {} /// @return as std::string std::string asString() const; /// conversion operator operator std::string() const; /// conversion to bool for checking if location was set operator bool() const; /// accessor to line int line() const { return line_; } /// accessor to file const char* file() const { return file_; } /// accessor to function const char* func() const { return func_; } private: // members int line_; const char* file_; const char* func_; protected: // methods void print(std::ostream&) const; }; // Macros #define Here() ::eckit::CodeLocation(__FILE__, __LINE__, __func__) //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/TimeStamp.cc0000664000175000017500000000260015161702250017661 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/eckit.h" #include "eckit/log/TimeStamp.h" #include "eckit/utils/Clock.h" namespace eckit { const char* TimeStamp::defaultFormat_ = "%Y-%m-%d %H:%M:%S"; TimeStamp::TimeStamp(const std::string& format) : time_(Clock::now()), format_(format) {} TimeStamp::TimeStamp(time_t t, const std::string& format) : time_(t), format_(format) {} std::ostream& operator<<(std::ostream& s, const TimeStamp& x) { if (x.format_ == "hex") { s << std::setw(16) << std::setfill('0') << std::hex << static_cast(x.time_); return s; } char buf[80]; #if eckit_HAVE_GMTIME_R struct tm t; ::strftime(buf, sizeof(buf), x.format_.c_str(), gmtime_r(&x.time_, &t)); #else ::strftime(buf, sizeof(buf), x.format_.c_str(), gmtime(&x.time_)); #endif s << buf; return s; } TimeStamp::operator std::string() const { std::ostringstream s; s << *this; return s.str(); } } // namespace eckit eckit-2.0.7/src/eckit/log/ChannelBuffer.h0000664000175000017500000000511715161702250020330 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file ChannelBuffer.h /// @author Baudouin Raoult /// @author Tiago Quintino /// @date August 2016 #ifndef eckit_log_ChannelBuffer_h #define eckit_log_ChannelBuffer_h #include #include #include "eckit/log/Channel.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class LogTarget; /// Stream buffer to be usedby Channel class ChannelBuffer : public std::streambuf { private: // methods /// constructor, taking ownership of stream ChannelBuffer(std::size_t size = 1024); ChannelBuffer(const ChannelBuffer&) = delete; ChannelBuffer& operator=(const ChannelBuffer&) = delete; ChannelBuffer(ChannelBuffer&&) = delete; ChannelBuffer& operator=(ChannelBuffer&&) = delete; ~ChannelBuffer() override; bool active() const; void reset(); void setTarget(LogTarget* target); void addTarget(LogTarget* target); void indent(const char* space = " "); void unindent(); void setStream(std::ostream& out); void addStream(std::ostream& out); void setFile(const std::string& path, size_t bufferSize = 4 * 1024); void addFile(const std::string& path, size_t bufferSize = 4 * 1024); void setCallback(channel_callback_t cb, void* data = nullptr); void addCallback(channel_callback_t cb, void* data = nullptr); protected: // methods /// override this to change buffer behavior /// @returns true if no error occured virtual bool dumpBuffer(); /// typically you don't need to override this /// @see dumpBuffer int_type overflow(int_type ch) override; /// typically you don't need to override this /// @see dumpBuffer int_type sync() override; protected: // members LogTarget* target_; std::vector buffer_; private: friend std::ostream& operator<<(std::ostream& os, const ChannelBuffer& c) { c.print(os); return os; } void print(std::ostream& s) const; friend class Channel; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Colour.h0000664000175000017500000000410215161702250017062 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Colour.h /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Nov 2011 #ifndef eckit_Colour_h #define eckit_Colour_h #include #include "eckit/eckit.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Colour { public: static std::ostream& on(std::ostream&); static std::ostream& off(std::ostream&); static std::ostream& reset(std::ostream&); static std::ostream& bold(std::ostream&); static std::ostream& underline(std::ostream&); static std::ostream& blink(std::ostream&); static std::ostream& reverse(std::ostream&); static std::ostream& hidden(std::ostream&); static std::ostream& black(std::ostream&); static std::ostream& red(std::ostream&); static std::ostream& green(std::ostream&); static std::ostream& yellow(std::ostream&); static std::ostream& magenta(std::ostream&); static std::ostream& blue(std::ostream&); static std::ostream& cyan(std::ostream&); static std::ostream& white(std::ostream&); static std::ostream& blackBackground(std::ostream&); static std::ostream& redBackground(std::ostream&); static std::ostream& greenBackground(std::ostream&); static std::ostream& yellowBackground(std::ostream&); static std::ostream& magentaBackground(std::ostream&); static std::ostream& blueBackground(std::ostream&); static std::ostream& cyanBackground(std::ostream&); static std::ostream& whiteBackground(std::ostream&); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Timer.h0000664000175000017500000000411215161702250016700 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Timer.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_Timer_h #define eckit_Timer_h #include #include #include "eckit/log/Log.h" namespace eckit { //----------------------------------------------------------------------------- class Timer { public: explicit Timer(); /// @param name of the timer, used for output /// @param o output stream to use for output explicit Timer(const std::string& name, std::ostream& o = Log::info()); /// @param name of the timer, used for output /// @param o output stream to use for output explicit Timer(const char* name, std::ostream& o = Log::info()); Timer(const Timer&) = delete; Timer& operator=(const Timer&) = delete; Timer(Timer&&) = delete; Timer& operator=(Timer&&) = delete; ~Timer(); void start(); void stop(); double elapsed(); double elapsed_cpu(); const std::string& name() const { return name_; } bool running() const { return !stopped_; } void report(const std::string& message = ""); void reset(const std::string& message = ""); protected: // methods void takeTime(); std::ostream& output() { return out_ << name_ << ": "; } private: // members std::string name_; bool stopped_; bool outputAtExit_; struct ::timeval timeStart_; struct ::timeval timeStop_; clock_t cpuStart_; clock_t cpuStop_; std::ostream& out_; }; //----------------------------------------------------------------------------- ::timeval operator-(const ::timeval&, const ::timeval&); //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/IndentTarget.h0000664000175000017500000000206715161702250020217 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file IndentTarget.h /// @author Baudouin Raoult #ifndef eckit_log_IndentTarget_h #define eckit_log_IndentTarget_h #include #include #include "eckit/log/PrefixTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class IndentTarget : public PrefixTarget { public: IndentTarget(const std::string& prefix, LogTarget* target, const char* space = " "); protected: void print(std::ostream& s) const; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/TraceTimer.h0000664000175000017500000000210015161702250017652 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Timer.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_TraceTimer_h #define eckit_TraceTimer_h #include "eckit/log/Log.h" #include "eckit/log/Timer.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- template class TraceTimer : public Timer { public: explicit TraceTimer(const char* name) : Timer(name, eckit::Log::debug()) {} explicit TraceTimer(const std::string& name) : Timer(name, eckit::Log::debug()) {} }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/SavedStatus.cc0000664000175000017500000000177015161702250020233 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/SavedStatus.h" #include "eckit/runtime/Monitor.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- SavedStatus::SavedStatus() : status_(Monitor::instance().status()) {} SavedStatus::~SavedStatus() { Monitor::instance().status(status_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/RotationTarget.h0000664000175000017500000000227315161702250020574 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file RotationTarget.h /// @author Tiago Quintino #ifndef eckit_log_RotationTarget_h #define eckit_log_RotationTarget_h #include #include "eckit/log/LogTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class RotationTarget : public LogTarget { public: // methods RotationTarget(const std::string& name = std::string()); ~RotationTarget() override; void write(const char* start, const char* end) override; void flush() override; protected: void print(std::ostream& s) const override; private: std::string name_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Timer.cc0000664000175000017500000000542315161702250017044 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/Seconds.h" #include "eckit/log/Timer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Timer::Timer() : name_("unnamed timer"), stopped_(true), outputAtExit_(false), cpuStart_(0), cpuStop_(0), out_(std::cout) { this->start(); } Timer::Timer(const std::string& name, std::ostream& o) : name_(name), stopped_(true), outputAtExit_(true), cpuStart_(0), cpuStop_(0), out_(o) { this->start(); } Timer::Timer(const char* name, std::ostream& o) : name_(name), stopped_(true), outputAtExit_(true), cpuStart_(0), cpuStop_(0), out_(o) { this->start(); } Timer::~Timer() { stop(); if (outputAtExit_) { report(); } } void Timer::start() { if (!running()) { ::gettimeofday(&timeStart_, nullptr); timeStop_ = timeStart_; cpuStart_ = ::clock(); cpuStop_ = cpuStart_; stopped_ = false; } } void Timer::stop() { if (!running()) { return; } takeTime(); stopped_ = true; } double Timer::elapsed() { if (running()) { takeTime(); } ::timeval diff = timeStop_ - timeStart_; return (double)diff.tv_sec + ((double)diff.tv_usec / 1000000.); } double Timer::elapsed_cpu() { if (running()) { takeTime(); } return double(cpuStop_ - cpuStart_) / CLOCKS_PER_SEC; } void Timer::report(const std::string& message) { const double s = elapsed(); const double cpu = elapsed_cpu(); out_ << (message.size() ? message : name_) << ": " << Seconds(s) << " elapsed, " << Seconds(cpu) << " cpu" << std::endl; } void Timer::reset(const std::string& message) { stop(); report(message); start(); } void Timer::takeTime() { cpuStop_ = ::clock(); ::gettimeofday(&timeStop_, nullptr); } //---------------------------------------------------------------------------------------------------------------------- ::timeval operator-(const ::timeval& a, const ::timeval& b) { ::timeval diff; diff.tv_sec = a.tv_sec - b.tv_sec; diff.tv_usec = a.tv_usec - b.tv_usec; if (diff.tv_usec < 0) { diff.tv_sec--; diff.tv_usec += 1000000; } return diff; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/MessageTarget.h0000664000175000017500000000202615161702250020355 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file MessageTarget.h /// @author Tiago Quintino #ifndef eckit_log_MessageTarget_h #define eckit_log_MessageTarget_h #include "eckit/log/LineBasedTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class MessageTarget : public LineBasedTarget { public: // methods MessageTarget(); private: void line(const char* line) override; void print(std::ostream& s) const override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/RotationTarget.cc0000664000175000017500000001020015161702250020717 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include #include "eckit/log/RotationTarget.h" #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/log/TimeStamp.h" #include "eckit/runtime/Main.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class RotationOutputStream; static StaticMutex local_mutex; class RotationOutputStream { public: static RotationOutputStream& instance(const std::string& name) { AutoLock lock(local_mutex); static std::map instances; auto j = instances.find(name); if (j == instances.end()) { instances[name] = new RotationOutputStream(name); j = instances.find(name); } return *(*j).second; } void write(const char* start, const char* end) { if (start >= end) return; AutoLock lock(local_mutex); rotout().write(start, end - start); } void flush() { AutoLock lock(local_mutex); if (last_) { last_->flush(); } } private: RotationOutputStream(const std::string& name) : name_(name), logfileFormat_(Resource("logfileFormat", "~/log/%Y-%m-%d/out")), buffer_(0), logFilesBufferSize_(Resource("logFilesBufferSize", 4 * 1024)) {} std::ostream& rotout() { time_t now = ::time(nullptr) / Time::secondsInDay; if (now != lastTime_ || last_ == nullptr) { TimeStamp ts(logfileFormat_); PathName path(ts); path.mkdir(0777); std::ostringstream os; os << path << "/" << name_; delete last_; last_ = new std::ofstream(); if (logFilesBufferSize_) { buffer_.resize(logFilesBufferSize_); buffer_.zero(); last_->rdbuf()->pubsetbuf(buffer_, buffer_.size()); } last_->open(os.str().c_str(), std::ofstream::out | std::ofstream::app); if (last_->fail()) { throw eckit::CantOpenFile(os.str()); } /// @todo Find a way to set the close on exec flags, /// or to get the file descriptor from ofstream so we can do: /// flags = fcntl(fd, F_GETFD); /// flags |= FD_CLOEXEC; /// fcntl(fd, F_SETFD, flags) lastTime_ = now; } return *last_; } private: // members std::ofstream* last_ = nullptr; time_t lastTime_ = 0; std::string name_; std::string logfileFormat_; Buffer buffer_; size_t logFilesBufferSize_; }; //---------------------------------------------------------------------------------------------------------------------- RotationTarget::RotationTarget(const std::string& name) : name_(name) { if (name_.empty()) { name_ = Main::instance().name(); } } RotationTarget::~RotationTarget() {} void RotationTarget::write(const char* start, const char* end) { if (start >= end) return; RotationOutputStream::instance(name_).write(start, end); } void RotationTarget::flush() { RotationOutputStream::instance(name_).flush(); } void RotationTarget::print(std::ostream& s) const { static std::string logfileFormat = Resource("logfileFormat", "~/log/%Y-%m-%d/out"); s << "RotationTarget(format=" << logfileFormat << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/FileTarget.cc0000664000175000017500000000324015161702250020005 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/FileTarget.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- FileTarget::FileTarget(const PathName& path, size_t bufferSize) : buffer_(bufferSize), path_(path) { if (bufferSize) { buffer_.resize(bufferSize); buffer_.zero(); out_.rdbuf()->pubsetbuf(buffer_, buffer_.size()); } out_.open(path_.asString().c_str(), std::ofstream::out | std::ofstream::app); if (!out_) { throw eckit::CantOpenFile(path_.asString()); } } FileTarget::~FileTarget() { // std::cerr << "FileTarget::~FileTarget() -- " << path_ << std::endl; // out_.close(); // if (out_) { // throw eckit::CloseError(path_.asString(), Here()); // } } void FileTarget::write(const char* start, const char* end) { if (start >= end) return; out_.write(start, end - start); } void FileTarget::flush() { out_.flush(); } void FileTarget::print(std::ostream& s) const { s << "FileTarget(path=" << path_ << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/Number.cc0000664000175000017500000000424215161702250017212 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/Number.h" namespace eckit::log { //---------------------------------------------------------------------------------------------------------------------- std::string Number::printBinary(W n) { std::stringstream ss; printBinary(ss, n); return ss.str(); } std::string Number::printHexadecimal(W n) { std::stringstream ss; printHexadecimal(ss, n); return ss.str(); } void Number::printHexadecimal(std::ostream& ss, W n) { ss << std::hex << n << std::dec; } void Number::printBinary(std::ostream& ss, W n) { unsigned char* s = reinterpret_cast(&n); bool oneSeen = false; int endianTest = 1; if (*reinterpret_cast(&endianTest)) { for (int i = sizeof(W) - 1; i >= 0; --i) { unsigned char c = s[i]; for (unsigned char mask = 1 << 7; mask; mask >>= 1) { if (c & mask) { ss << '1'; oneSeen = true; } else { if (oneSeen) { ss << '0'; } } } } } else { for (size_t i = 0; i < sizeof(W); ++i) { unsigned char c = s[i]; for (unsigned char mask = 1 << 7; mask; mask >>= 1) { if (c & mask) { ss << '1'; oneSeen = true; } else { if (oneSeen) { ss << '0'; } } } } } if (!oneSeen) { ss << '0'; } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::log eckit-2.0.7/src/eckit/log/ETA.h0000664000175000017500000000214215161702250016232 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File ETA.h // Baudouin Raoult - ECMWF Jul 96 #ifndef eckit_ETA_h #define eckit_ETA_h #include #include namespace eckit { //-------------------------------------------------------------------------------------------------- class ETA { public: // -- Contructors ETA(double); ETA(const struct ::timeval&); // -- Operators operator std::string() const; friend std::ostream& operator<<(std::ostream&, const ETA&); private: // There is no private copy constructor as this will confuse g++ 4.x.x // -- Members double ETA_; }; //-------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/TeeTarget.cc0000664000175000017500000000313515161702250017646 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/log/OStreamTarget.h" #include "eckit/log/TeeTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- TeeTarget::TeeTarget(LogTarget* left, LogTarget* right) : left_(left), right_(right) { if (left_) { left_->attach(); } if (right_) { right_->attach(); } } TeeTarget::~TeeTarget() { if (left_) { left_->detach(); } if (right_) { right_->detach(); } } void TeeTarget::write(const char* start, const char* end) { if (start >= end) return; if (left_) { left_->write(start, end); } if (right_) { right_->write(start, end); } } void TeeTarget::flush() { if (left_) { left_->flush(); } if (right_) { right_->flush(); } } void TeeTarget::print(std::ostream& s) const { s << "TeeTarget("; if (left_) { s << *left_; } if (right_) { s << *right_; } s << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/MonitorTarget.cc0000664000175000017500000000223715161702250020562 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/MonitorTarget.h" #include "eckit/runtime/Monitor.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- MonitorTarget::MonitorTarget(LogTarget* target) : WrapperTarget(target) {} MonitorTarget::~MonitorTarget() {} void MonitorTarget::write(const char* start, const char* end) { if (start >= end) return; Monitor::instance().out(const_cast(start), const_cast(end)); target_->write(start, end); } void MonitorTarget::print(std::ostream& s) const { s << "MonitorTarget()"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/ColouringTarget.cc0000664000175000017500000000275115161702250021075 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/log/ColouringTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ColouringTarget::ColouringTarget(LogTarget* target, ColouringTarget::colourproc begin, ColouringTarget::colourproc end) : WrapperTarget(target) { std::ostringstream beginss; beginss << *begin; begin_ = beginss.str(); std::ostringstream endss; endss << *end; end_ = endss.str(); } ColouringTarget::~ColouringTarget() {} void ColouringTarget::writePrefix() { target_->write(begin_.c_str(), begin_.c_str() + begin_.size()); } void ColouringTarget::writeSuffix() { target_->write(end_.c_str(), end_.c_str() + end_.size()); } void ColouringTarget::print(std::ostream& s) const { s << "ColouringTarget("; if (target_) { s << "target=" << *target_; } s << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/Plural.h0000664000175000017500000000231715161702250017064 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_Plural_h #define eckit_Plural_h #include #include "eckit/log/BigNum.h" namespace eckit { class Plural { public: // methods Plural(int count, const std::string& s) : s_(s), count_(count) {} Plural(const Plural&) = delete; Plural& operator=(const Plural&) = delete; Plural(Plural&&) = delete; Plural& operator=(Plural&&) = delete; ~Plural() {} protected: // methods void print(std::ostream& s) const { s << BigNum(count_) << ' ' << s_; if (count_ > 1) { s << 's'; } } private: // members friend std::ostream& operator<<(std::ostream& s, const Plural& p) { p.print(s); return s; } private: // members std::string s_; int count_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Log.cc0000664000175000017500000002643515161702250016513 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Channel.h" #include "eckit/log/FileTarget.h" #include "eckit/log/Log.h" #include "eckit/log/MessageTarget.h" #include "eckit/log/OStreamTarget.h" #include "eckit/log/PrefixTarget.h" #include "eckit/log/StatusTarget.h" #include "eckit/log/UserChannel.h" #include "eckit/runtime/Main.h" #include "eckit/system/LibraryManager.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/ThreadSingleton.h" #include "eckit/utils/Translator.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- #if defined(_GNU_SOURCE) /* To use with GNU libc strerror_r */ static void handle_strerror_r(std::ostream& s, int e, char[], char* p) { if (p) { s << " (" << p << ")"; } else { s << " (errno = " << e << ") "; } } #elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 || _POSIX_VERSION >= 200112L || _XOPEN_VERSION >= 600) /* To use with XSI-compliant strerror_r * * Linux defines _POSIX_C_SOURCE and _XOPEN_SOURCE * BSD defines _POSIX_VERSION and _XOPEN_VERSION * glibc defines _GNU_SOURCE and implements a non-XSI compliant strerror_r */ static void handle_strerror_r(std::ostream& s, int e, char es[], int hs) { if (hs == 0) { s << " (" << es << ") "; } else { s << " (errno = " << e << ") "; } } #else /* we don't know what to do */ #warning "eckit doesn't recognise stderror_r since this isn't a GNU libc or a supported UNIX" /* This uses the deprecated sys_errlist[] error arrays */ static void handle_strerror_r(std::ostream& s, int e, ...) { if (e < sys_nerr) s << " (" << sys_errlist[e] << ") "; else s << " (errno = " << e << ") "; } #endif //---------------------------------------------------------------------------------------------------------------------- struct CreateStatusChannel { Channel* operator()() { return new Channel(new StatusTarget()); } }; std::ostream& Log::status() { static ThreadSingleton x; return x.instance(); } struct CreateMessageChannel { Channel* operator()() { return new Channel(new MessageTarget()); } }; std::ostream& Log::message() { static ThreadSingleton x; return x.instance(); } struct CreateLogChannel { virtual Channel* createChannel() = 0; Channel* operator()() { try { return createChannel(); } catch (std::exception& e) { std::cerr << "Exception caught when creating channel: " << e.what() << std::endl; return new Channel(new OStreamTarget(std::cout)); } } }; struct CreateMetricsChannel : public CreateLogChannel { virtual Channel* createChannel() { return new Channel(Main::instance().createMetricsLogTarget()); } }; Channel& Log::metrics() { if (!Main::ready()) { static Channel empty(new PrefixTarget("PRE-MAIN-METRICS", new OStreamTarget(std::cout))); return empty; } static ThreadSingleton x; return x.instance(); } struct CreateInfoChannel : public CreateLogChannel { virtual Channel* createChannel() { return new Channel(Main::instance().createInfoLogTarget()); } }; Channel& Log::info() { if (!Main::ready()) { static Channel empty(new PrefixTarget("PRE-MAIN-INFO", new OStreamTarget(std::cout))); return empty; } static ThreadSingleton x; return x.instance(); } struct CreateErrorChannel : public CreateLogChannel { virtual Channel* createChannel() { return new Channel(Main::instance().createErrorLogTarget()); } }; Channel& Log::error() { if (!Main::ready()) { static Channel empty(new PrefixTarget("PRE-MAIN-ERROR", new OStreamTarget(std::cout))); return empty; } static ThreadSingleton x; return x.instance(); } struct CreateWarningChannel : public CreateLogChannel { virtual Channel* createChannel() { return new Channel(Main::instance().createWarningLogTarget()); } }; Channel& Log::warning() { if (!Main::ready()) { static Channel empty(new PrefixTarget("PRE-MAIN-WARNING", new OStreamTarget(std::cout))); return empty; } static ThreadSingleton x; return x.instance(); } struct CreateDebugChannel : public CreateLogChannel { virtual Channel* createChannel() { return new Channel(Main::instance().createDebugLogTarget()); } }; Channel& Log::debug() { if (!Main::ready()) { const char* e = getenv("DEBUG"); if (e && bool(Translator()(e))) { static Channel empty(new PrefixTarget("PRE-MAIN-DEBUG", new OStreamTarget(std::cout))); return empty; } static Channel empty; return empty; } if (!Main::instance().debug_) { static ThreadSingleton empty; return empty.instance(); } static ThreadSingleton x; return x.instance(); } std::ostream& Log::panic() { try { return Log::error(); } catch (std::exception&) { return std::cerr; } } UserChannel& Log::user() { static ThreadSingleton x; return x.instance(); } std::ostream& Log::userInfo() { UserChannel& u = user(); u.msgType(UserChannel::INFO); return u; } std::ostream& Log::userError() { UserChannel& u = user(); u.msgType(UserChannel::ERROR); return u; } std::ostream& Log::userWarning() { UserChannel& u = user(); u.msgType(UserChannel::WARN); return u; } void Log::notifyClient(const std::string& msg) { UserChannel& u = user(); UserMsg* um = u.userMsg(); if (um) { um->notifyClient(msg); } } //---------------------------------------------------------------------------------------------------------------------- void Log::setStream(std::ostream& out) { info().setStream(out); warning().setStream(out); error().setStream(out); if (debug()) { debug().setStream(out); } std::vector libs = eckit::system::LibraryManager::list(); for (std::vector::iterator libname = libs.begin(); libname != libs.end(); ++libname) { system::LibraryManager::lookup(*libname).debugChannel().setStream(out); } } void Log::addStream(std::ostream& out) { info().addStream(out); warning().addStream(out); error().addStream(out); if (debug()) { debug().addStream(out); } std::vector libs = eckit::system::LibraryManager::list(); for (std::vector::iterator libname = libs.begin(); libname != libs.end(); ++libname) { system::LibraryManager::lookup(*libname).debugChannel().addStream(out); } } void Log::setFile(const std::string& path) { LogTarget* file = new FileTarget(path); info().setTarget(file); warning().setTarget(file); error().setTarget(file); if (debug()) { debug().setTarget(file); } std::vector libs = eckit::system::LibraryManager::list(); for (std::vector::iterator libname = libs.begin(); libname != libs.end(); ++libname) { system::LibraryManager::lookup(*libname).debugChannel().setTarget(file); } } void Log::addFile(const std::string& path) { LogTarget* file = new FileTarget(path); info().addTarget(file); warning().addTarget(file); error().addTarget(file); if (debug()) { debug().addTarget(file); } std::vector libs = eckit::system::LibraryManager::list(); for (std::vector::iterator libname = libs.begin(); libname != libs.end(); ++libname) { system::LibraryManager::lookup(*libname).debugChannel().addTarget(file); } } void Log::setCallback(channel_callback_t cb, void* data) { info().setCallback(cb, data); warning().setCallback(cb, data); error().setCallback(cb, data); if (debug()) { debug().setCallback(cb, data); } std::vector libs = eckit::system::LibraryManager::list(); for (std::vector::iterator libname = libs.begin(); libname != libs.end(); ++libname) { if (system::LibraryManager::lookup(*libname).debugChannel()) { system::LibraryManager::lookup(*libname).debugChannel().setCallback(cb, data); } } } void Log::addCallback(channel_callback_t cb, void* data) { info().addCallback(cb, data); warning().addCallback(cb, data); error().addCallback(cb, data); if (debug()) { debug().addCallback(cb, data); } std::vector libs = eckit::system::LibraryManager::list(); for (std::vector::iterator libname = libs.begin(); libname != libs.end(); ++libname) { system::LibraryManager::lookup(*libname).debugChannel().addCallback(cb, data); } } void Log::flush() { info().flush(); warning().flush(); error().flush(); debug().flush(); std::vector libs = eckit::system::LibraryManager::list(); for (std::vector::iterator libname = libs.begin(); libname != libs.end(); ++libname) { system::LibraryManager::lookup(*libname).debugChannel().flush(); } } void Log::reset() { info().reset(); warning().reset(); error().reset(); debug().reset(); std::vector libs = eckit::system::LibraryManager::list(); for (std::vector::iterator libname = libs.begin(); libname != libs.end(); ++libname) { system::LibraryManager::lookup(*libname).debugChannel().reset(); } } void Log::print(std::ostream& os) { os << "Log::info() " << info() << std::endl; os << "Log::warning() " << warning() << std::endl; os << "Log::error() " << error() << std::endl; os << "Log::debug() " << debug() << std::endl; std::vector libs = eckit::system::LibraryManager::list(); for (std::vector::iterator libname = libs.begin(); libname != libs.end(); ++libname) { os << *libname << ".debug() " << system::LibraryManager::lookup(*libname).debugChannel() << std::endl; } } //---------------------------------------------------------------------------------------------------------------------- std::ostream& Log::syserr(std::ostream& s) { int e = errno; char estr[256]; handle_strerror_r(s, e, estr, strerror_r(e, estr, sizeof(estr))); return s; } template class ThreadSingleton; //---------------------------------------------------------------------------------------------------------------------- static int xindex = std::ios::xalloc(); int format(std::ostream& s) { return s.iword(xindex); } void format(std::ostream& s, int f) { s.iword(xindex) = f; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/Progress.h0000664000175000017500000000354015161702250017430 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Progress.h // Baudouin Raoult - ECMWF Nov 96 #ifndef eckit_Progress_h #define eckit_Progress_h #include //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class Progress { public: // -- Exceptions // None // -- Contructors Progress(const std::string&, unsigned long long, unsigned long long); // -- Destructor ~Progress(); // -- Convertors // None // -- Operators void operator()(unsigned long long); // -- Methods // None // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods // void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed Progress(const Progress&); Progress& operator=(const Progress&); // -- Members // None // -- Methods // None // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends // friend std::ostream& operator<<(std::ostream& s,const Progress& p) // { p.print(s); return s; } }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Statistics.cc0000664000175000017500000001411515161702250020114 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "Statistics.h" #include #include #include #include #include "eckit/log/BigNum.h" #include "eckit/log/Bytes.h" #include "eckit/log/Seconds.h" #include "eckit/serialisation/Stream.h" namespace eckit { const size_t WIDTH = 34; Timer Statistics::timer_; //---------------------------------------------------------------------------------------------------------------------- void Statistics::reportCount(std::ostream& out, const char* title, size_t value, const char* indent, bool always) { if (value || always) { out << indent << title << std::setw(WIDTH - strlen(title)) << " : " << eckit::BigNum(value) << std::endl; } } void Statistics::reportUnit(std::ostream& out, const char* title, const char* unit, double value, const char* indent, bool always) { if (value || always) { out << indent << title << std::setw(WIDTH - strlen(title)) << " : " << value << " " << unit << std::endl; } } void Statistics::reportRate(std::ostream& out, const char* title, unsigned long long value, const char* indent, bool always) { if (value || always) { out << indent << title << std::setw(WIDTH - strlen(title)) << " : " << eckit::BigNum(value) << " bytes/s (" << eckit::Bytes(value) << "/s)" << std::endl; } } void Statistics::reportRate(std::ostream& out, const std::string& title, size_t bytes, double elapsed, const char* indent, bool always) { if (bytes || always) { double rate = 0; if (bytes != 0 and elapsed > 0) { rate = bytes / elapsed; } out << indent << title << std::setw(WIDTH - title.length()) << " : " << BigNum(size_t(rate)) << " bytes/s" << " (" << Bytes(rate) << " per second)" << std::endl; } } void Statistics::reportBytes(std::ostream& out, const char* title, unsigned long long value, const char* indent, bool always) { if (value || always) { out << indent << title << std::setw(WIDTH - strlen(title)) << " : " << eckit::BigNum(value) << " (" << eckit::Bytes(value) << ")" << std::endl; } } void Statistics::reportBytesStats(std::ostream& out, const std::string& title, size_t count, size_t bytes, size_t sumsquared, const char* indent, bool always) { if (count || always) { double average = 0; double stdDeviation = 0; if (count != 0) { average = bytes / count; stdDeviation = std::sqrt(std::max((count * sumsquared) - (bytes * bytes), size_t(0))) / count; } out << indent << title << std::setw(WIDTH - title.length()) << " (tot, avg, std dev) : " << BigNum(bytes) << " (" << Bytes(bytes) << ")" << ", " << BigNum(size_t(average)) << " (" << Bytes(average) << ")" << ", " << BigNum(size_t(stdDeviation)) << " (" << Bytes(stdDeviation) << ")" << std::endl; } } void Statistics::reportTime(std::ostream& out, const char* title, const Timing& value, const char* indent, bool always) { if (value.updates_ || always) { out << indent << title << std::setw(WIDTH - strlen(title)) << " : " << eckit::Seconds(value.elapsed_) << " (" << eckit::Seconds(value.cpu_) << " CPU). Updates: " << eckit::BigNum(value.updates_) << std::endl; } } void Statistics::reportTime(std::ostream& out, const char* title, double value, const char* indent, bool always) { if (value || always) { out << indent << title << std::setw(WIDTH - strlen(title)) << " : " << eckit::Seconds(value) << std::endl; } } void Statistics::reportTimeStats(std::ostream& out, const std::string& title, size_t count, double sum_times, double sum_times_squared, const char* indent, bool always) { if (count || always) { double average = 0; double stdDeviation = 0; if (count != 0) { average = sum_times / count; stdDeviation = std::sqrt(std::max((count * sum_times_squared) - (sum_times * sum_times), 0.0)) / count; } out << indent << title << std::setw(WIDTH - title.length()) << " (tot, avg, std dev) : " << sum_times << " s" << ", " << average << " s" << ", " << stdDeviation << " s" << std::endl; } } //---------------------------------------------------------------------------------------------------------------------- Timing& Timing::operator+=(const Timing& other) { elapsed_ += other.elapsed_; cpu_ += other.cpu_; updates_ += other.updates_; return *this; } Timing& Timing::operator-=(const Timing& other) { elapsed_ -= other.elapsed_; cpu_ -= other.cpu_; // We don't remove the number of update because // we use this operator to remove sub-timers // updates_ -= other.updates_; return *this; } Timing& Timing::operator/=(size_t n) { elapsed_ /= n; cpu_ /= n; if (updates_) { updates_ /= n; if (!updates_) { updates_ = 1; } } return *this; } Timing Timing::operator-(const Timing& other) const { return Timing(elapsed_ - other.elapsed_, cpu_ - other.cpu_, 1); } Stream& operator<<(Stream& s, const Timing& t) { s << t.elapsed_; s << t.cpu_; s << t.updates_; return s; } Stream& operator>>(Stream& s, Timing& t) { s >> t.elapsed_; s >> t.cpu_; s >> t.updates_; return s; } std::ostream& operator<<(std::ostream& s, const Timing& t) { s << t.elapsed_; return s; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/TeeTarget.h0000664000175000017500000000233115161702250017505 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file TeeTarget.h /// @author Baudouin Raoult /// @author Tiago Quintino /// @date August 2016 #ifndef eckit_log_TeeTarget_h #define eckit_log_TeeTarget_h #include "eckit/log/LogTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class TeeTarget : public LogTarget { public: // methods TeeTarget(LogTarget* left, LogTarget* right); ~TeeTarget() override; protected: void print(std::ostream& s) const override; private: LogTarget* left_; LogTarget* right_; private: void write(const char* start, const char* end) override; void flush() override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Number.h0000664000175000017500000000211515161702250017051 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Aug 2018 #ifndef eckit_log_Number_H #define eckit_log_Number_H #include #include namespace eckit::log { //---------------------------------------------------------------------------------------------------------------------- class Number { public: using W = long long; static void printBinary(std::ostream&, W); static std::string printBinary(W); static void printHexadecimal(std::ostream&, W); static std::string printHexadecimal(W); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::log #endif eckit-2.0.7/src/eckit/log/CodeLocation.cc0000664000175000017500000000234415161702250020326 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/log/CodeLocation.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- std::string CodeLocation::asString() const { std::ostringstream oss; print(oss); return oss.str(); } eckit::CodeLocation::operator std::string() const { return asString(); } CodeLocation::operator bool() const { return file_ && ::strlen(file_); } void CodeLocation::print(std::ostream& os) const { if (file_) { os << " (" << file_ << ":" << line_; if (func_ && ::strlen(func_) > 0) { os << " " << func_; } os << ")"; } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/Progress.cc0000664000175000017500000000223215161702250017563 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Progress.h" #include "eckit/runtime/Monitor.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Progress::Progress(const std::string& name, unsigned long long min, unsigned long long max) { Monitor::instance().progress(name, min, max); } Progress::~Progress() { Monitor::instance().progress(); } void Progress::operator()(unsigned long long value) { Monitor::instance().progress(value); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/LineBasedTarget.cc0000664000175000017500000000336715161702250020766 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/maths/Functions.h" #include "eckit/log/LineBasedTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- LineBasedTarget::LineBasedTarget() : size_(4096), buffer_(new char[size_]), position_(0) { ASSERT(buffer_); } LineBasedTarget::~LineBasedTarget() { delete[] buffer_; } void LineBasedTarget::reserve(size_t size) { if (size_ < size) { delete[] buffer_; size_ = round(size, 1024 * 1024); buffer_ = new char[size_]; ASSERT(buffer_); } } void LineBasedTarget::write(const char* start, const char* end) { if (start >= end) return; reserve(position_ + (end - start) + 1); while (start < end) { if (*start == '\n') { buffer_[position_] = 0; line(buffer_); position_ = 0; start++; } else { buffer_[position_++] = *start++; } } } void LineBasedTarget::flush() { // LineBasedTarget doesn't flush() since the concrete classes treat each line independently // and upon write(), and often don't require further flushing } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/CallbackTarget.h0000664000175000017500000000237015161702250020467 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file CallbackTarget.h /// @author Tiago Quintino #ifndef eckit_log_CallbackTarget_h #define eckit_log_CallbackTarget_h #include #include #include "eckit/log/LineBasedTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class CallbackTarget : public LineBasedTarget { public: using callback_t = void (*)(void* ctxt, const char* msg); CallbackTarget(callback_t callback, void* context = nullptr); ~CallbackTarget() override; private: void line(const char* line) override; void print(std::ostream& s) const override; private: callback_t callback_; void* context_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/SysLogTCPTarget.cc0000664000175000017500000000221315161702250020714 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/SysLogTCPTarget.h" #include #include "eckit/log/SysLog.h" #include "eckit/net/TCPSocket.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- SysLogTCPTarget::SysLogTCPTarget(net::TCPSocket& socket, SysLog log) : TCPTarget(socket), log_(std::move(log)) {} void SysLogTCPTarget::msgid(int msg_id) { log_.msgid(msg_id); } void SysLogTCPTarget::write(const char* start, const char* end) { log_.message({start, end}); TCPTarget::write(log_); } void SysLogTCPTarget::flush() { // no-op } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/CallbackTarget.cc0000664000175000017500000000214715161702250020627 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/CallbackTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- CallbackTarget::CallbackTarget(callback_t callback, void* context) : callback_(callback), context_(context) {} CallbackTarget::~CallbackTarget() { // std::cerr << "CallbackTarget::~CallbackTarget()" << std::endl; } void CallbackTarget::line(const char* line) { callback_(context_, line); } void CallbackTarget::print(std::ostream& s) const { s << "CallbackTarget()"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/SysLog.h0000664000175000017500000000640615161702250017050 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @date Feb 2018 #ifndef eckit_log_SysLog_h #define eckit_log_SysLog_h #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// SysLog implements the RFC 5424 format for logging class SysLog { public: enum Facility { Kernel = 0, User, Mail, SystemDaemon, Security, SysLogD, LinePrinter, NetworkNews, UUCP, Clock, Security2, FTP, NTP, LogAudit, LogAlert, Clock2, Local0, Local1, Local2, Local3, Local4, Local5, Local6, Local7, //< we use this for MARS }; enum Severity { Emergency = 0, Alert, Critical, Error, Warning, Notice, Info, Debug }; public: // methods static char nilvalue() { return '-'; } explicit SysLog(std::string msg, int msgid = 0, Facility f = SysLog::User, Severity s = SysLog::Info); explicit SysLog(int msgid = 0, Facility f = SysLog::User, Severity s = SysLog::Info); unsigned priority() const { return facility_ * 8 + severity_; } unsigned version() const { return 1; } std::string timestamp() const { return timestamp_; } std::string fqdn() const; std::string appName() const; void appName(const std::string&); /// Not necessarily a pid, but for now we settled with that /// Semantics is application dependent int procid() const; /// Follows some application classification therefore semantics is application dependent int msgid() const { return msgid_; } /// Sets the message ID void msgid(int msg_id); /// Sets the message (updates timestamp) void message(std::string msg); std::string structuredData() const; operator std::string() const; friend std::ostream& operator<<(std::ostream& s, const SysLog& o) { o.print(s); return s; } /// Optional fields for structured data (RFC 5424 section 6.3) void software(const std::string& software) { software_ = software; } void swVersion(const std::string& version) { swVersion_ = version; } void enterpriseId(const std::string& id) { enterpriseId_ = id; } private: // methods void print(std::ostream& out) const; void updateTimestamp(); private: // members Facility facility_; Severity severity_; std::string timestamp_; std::string appName_; int msgid_; std::string msg_; // optional fields for structured data std::string software_; std::string swVersion_; std::string enterpriseId_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/SysLog.cc0000664000175000017500000000536415161702250017210 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/SysLog.h" #include #include #include #include #include "eckit/log/TimeStamp.h" #include "eckit/net/IPAddress.h" #include "eckit/runtime/Main.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- SysLog::SysLog(std::string msg, int msgid, Facility f, Severity s) : facility_(f), severity_(s), appName_(Main::instance().name()), msgid_(msgid), msg_(std::move(msg)) { updateTimestamp(); } SysLog::SysLog(int msgid, Facility f, Severity s) : SysLog("", msgid, f, s) {} std::string SysLog::fqdn() const { return Main::hostname(); } std::string SysLog::appName() const { return appName_; } void SysLog::appName(const std::string& app) { appName_ = app; } int SysLog::procid() const { return ::getpid(); } void SysLog::updateTimestamp() { timestamp_ = TimeStamp("%Y-%m-%dT%H:%M:%SZ"); ///< assumes we are in UTC } void SysLog::msgid(int msg_id) { msgid_ = msg_id; } void SysLog::message(std::string msg) { msg_ = std::move(msg); updateTimestamp(); } std::string SysLog::structuredData() const { if (software_.empty() && swVersion_.empty() && enterpriseId_.empty()) { return std::string(1, nilvalue()); } // RFC 5424 section 6.3 (only origin) std::ostringstream s; std::string ip = net::IPAddress::myIPAddress().asString(); s << "[origin ip=\"" << ip << "\""; if (!enterpriseId_.empty()) { s << " enterpriseId=\"" << enterpriseId_ << "\""; } if (!software_.empty()) { s << " software=\"" << software_ << "\""; if (!swVersion_.empty()) { s << " swVersion=\"" << swVersion_ << "\""; } } s << "]"; return s.str(); } SysLog::operator std::string() const { std::ostringstream os; static char sep = ' '; os // RFC 5424 section 6.2 (Header) << "<" << priority() << ">" << version() << sep << timestamp() << sep << fqdn() << sep << appName() << sep << procid() << sep << msgid() << sep // RFC 5424 section 6.3 << structuredData() << sep // RFC 5424 section 6.4 << msg_; return os.str(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/WrapperTarget.cc0000664000175000017500000000332015161702250020545 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/log/OStreamTarget.h" #include "eckit/log/WrapperTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- WrapperTarget::WrapperTarget(LogTarget* target) : target_(target), prefix_(true) { if (!target_) { target_ = new OStreamTarget(std::cout); } target_->attach(); } WrapperTarget::~WrapperTarget() { target_->detach(); } void WrapperTarget::write(const char* start, const char* end) { const char* begin = start; while (start < end) { if (*start == '\n') { target_->write(begin, start); writeSuffix(); target_->write(start, start + 1); prefix_ = true; begin = start + 1; } else { if (prefix_) { writePrefix(); prefix_ = false; } } start++; } if (begin < end) { if (prefix_) { writePrefix(); prefix_ = false; } target_->write(begin, end); } } void WrapperTarget::flush() { target_->flush(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/TimeStampTarget.cc0000664000175000017500000000301515161702250021031 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/log/TimeStamp.h" #include "eckit/log/TimeStampTarget.h" #include "eckit/runtime/Monitor.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- TimeStampTarget::TimeStampTarget(const char* tag, LogTarget* target) : WrapperTarget(target), tag_(tag) {} TimeStampTarget::~TimeStampTarget() {} void TimeStampTarget::writePrefix() { std::ostringstream oss; oss << std::setw(3) << std::setfill('0') << Monitor::instance().self() << std::setfill(' ') << ' ' << TimeStamp() << ' '; if (tag_ && *tag_) { oss << tag_ << ' '; } std::string s = oss.str(); const char* p = s.c_str(); target_->write(p, p + s.size()); } void TimeStampTarget::writeSuffix() {} void TimeStampTarget::print(std::ostream& s) const { s << "TimeStampTarget("; if (target_) { s << "target=" << *target_ << ")"; } s << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/LogTarget.h0000664000175000017500000000236415161702250017517 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file LogTarget.h /// @author Tiago Quintino #ifndef eckit_log_LogTarget_h #define eckit_log_LogTarget_h #include #include "eckit/memory/Counted.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class LogTarget : public Counted { public: // methods LogTarget(); virtual void write(const char* start, const char* end) = 0; virtual void flush() = 0; ~LogTarget() override; protected: friend std::ostream& operator<<(std::ostream& os, const LogTarget& c) { c.print(os); return os; } virtual void print(std::ostream& s) const; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/LineBasedTarget.h0000664000175000017500000000226615161702250020625 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file LineBasedTarget.h /// @author Tiago Quintino #ifndef eckit_log_LineBasedTarget_h #define eckit_log_LineBasedTarget_h #include "eckit/log/LogTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class LineBasedTarget : public LogTarget { protected: // methods LineBasedTarget(); ~LineBasedTarget(); private: void write(const char* start, const char* end) override; void flush() override; virtual void line(const char* line) = 0; void reserve(size_t size); size_t size_; char* buffer_; size_t position_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/ColouringTarget.h0000664000175000017500000000257015161702250020736 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file ColouringTarget.h /// @author Baudouin Raoult /// @author Tiago Quintino /// @date August 2016 #ifndef eckit_log_ColouringTarget_h #define eckit_log_ColouringTarget_h #include #include #include "eckit/log/Colour.h" #include "eckit/log/WrapperTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class ColouringTarget : public WrapperTarget { public: using colourproc = std::ostream& (*)(std::ostream&); ColouringTarget(LogTarget* target, colourproc begin, colourproc end = &Colour::reset); ~ColouringTarget() override; protected: void print(std::ostream& s) const override; private: void writePrefix() override; void writeSuffix() override; std::string begin_; std::string end_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/TimeStampTarget.h0000664000175000017500000000227215161702250020677 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file TimeStampTarget.h /// @author Tiago Quintino #ifndef eckit_log_TimeStampTarget_h #define eckit_log_TimeStampTarget_h #include #include "eckit/log/WrapperTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class TimeStampTarget : public WrapperTarget { public: TimeStampTarget(const char* tag = "", LogTarget* target = nullptr); ~TimeStampTarget() override; protected: void print(std::ostream& s) const override; private: void writePrefix() override; void writeSuffix() override; private: const char* tag_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/ProgressTimer.h0000664000175000017500000000372315161702250020434 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File ProgressTimer.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_ProgressTimer_h #define eckit_ProgressTimer_h #include "eckit/log/Log.h" #include "eckit/log/Timer.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class ProgressTimer : public Timer { public: /// @param name of the timer, used for output /// @param limit counter maximum value /// @param unit counter unit (singular) /// @param countedProgress how often to output progress, based on count /// @param o output stream ProgressTimer(const std::string& name, size_t limit, const std::string& unit, size_t progressCounted = 10000, std::ostream& o = Log::info()); /// @param name of the timer, used for output /// @param limit counter maximum value /// @param unit counter unit (singular) /// @param timedProgress how often to output progress, based on elapsed time /// @param o output stream ProgressTimer(const std::string& name, size_t limit, const std::string& unit, double progressTimed = 10., std::ostream& o = Log::info()); ProgressTimer& operator++(); operator bool() const; private: // members const size_t limit_; const std::string unit_; const size_t progressCounted_; const double progressTimed_; size_t counter_; double lastTime_; bool hasOutput_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/SysLogTCPTarget.h0000664000175000017500000000360115161702250020560 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file SysLogTCPTarget.h /// @author Metin Cakircali /// @date July 2025 #include "eckit/log/SysLog.h" #include "eckit/log/TCPTarget.h" namespace eckit { namespace net { class TCPSocket; } //---------------------------------------------------------------------------------------------------------------------- /// sends SysLog (RFC 5424 format) logs over TCP class SysLogTCPTarget : public TCPTarget { /// Example SysLog: /// SysLog log(SysLog::Local7, SysLog::Info); /// log.enterpriseId("7464"); // ECMWF's id /// log.appName("mars-client-cpp"); /// log.software("mars"); /// log.swVersion("0.0.1"); public: // methods /// @param socket TCP socket to use for log messages /// @param log SysLog object that hold application specific information /// @note takes ownership of [socket] explicit SysLogTCPTarget(net::TCPSocket& socket, SysLog log); // rules SysLogTCPTarget(const SysLogTCPTarget&) = delete; SysLogTCPTarget& operator=(const SysLogTCPTarget&) = delete; SysLogTCPTarget(SysLogTCPTarget&&) = delete; SysLogTCPTarget& operator=(SysLogTCPTarget&&) = delete; ~SysLogTCPTarget() override = default; void write(const char* start, const char* end) override; void flush() override; void msgid(int msg_id); private: // members SysLog log_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/IndentTarget.cc0000664000175000017500000000207215161702250020351 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/IndentTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- IndentTarget::IndentTarget(const std::string& prefix, LogTarget* target, const char* space) : PrefixTarget(" " + prefix, target, space) {} void IndentTarget::print(std::ostream& s) const { s << "IndentTarget(prefix=" << prefix_ << ", space=" << space_; if (target_) { s << ", target=" << *target_; } s << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/PrefixTarget.h0000664000175000017500000000250415161702250020227 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file PrefixTarget.h /// @author Baudouin Raoult /// @author Tiago Quintino /// @date August 2016 #ifndef eckit_log_PrefixTarget_h #define eckit_log_PrefixTarget_h #include #include #include #include "eckit/log/WrapperTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class PrefixTarget : public WrapperTarget { public: PrefixTarget(const std::string& prefix, LogTarget* target = nullptr, const char* space = " "); private: virtual void writePrefix(); virtual void writeSuffix(); protected: void print(std::ostream& s) const; protected: std::string prefix_; const char* space_; size_t prefixLength_; size_t spaceLength_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/MonitorTarget.h0000664000175000017500000000227715161702250020430 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file MonitorTarget.h /// @author Tiago Quintino #ifndef eckit_log_MonitorTarget_h #define eckit_log_MonitorTarget_h #include #include #include "eckit/log/WrapperTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class MonitorTarget : public WrapperTarget { public: MonitorTarget(LogTarget* target = nullptr); ~MonitorTarget() override; private: void write(const char* start, const char* end) override; void writePrefix() override {} void writeSuffix() override {} void print(std::ostream& s) const override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Colour.cc0000664000175000017500000001017115161702250017223 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/config/Resource.h" #include "eckit/log/Colour.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static int xindex = std::ios::xalloc(); enum { RESET = 0, BOLD = 1, UNDERLINE = 4, BLINK = 5, REVERSE = 7, HIDDEN = 8 }; enum { BLACK = 0, RED = 1, GREEN = 2, YELLOW = 3, BLUE = 4, MAGENTA = 5, CYAN = 6, WHITE = 7 }; // #define X(a) putchar(a) // #define X(a) out << (a) #define X(a) out << char(a) static bool connected_to_console() { int save_errno = errno; bool result = ::isatty(1); // This may change errno errno = save_errno; return result; } static std::ostream& put(std::ostream& out, int fg, int bg, int attr) { static bool colourOutput = Resource("$ECKIT_COLOUR_OUTPUT;-colour;colourOutput", connected_to_console()); if (colourOutput && out.iword(xindex) == 0) { int n = 0; X(char(0x1B)); X('['); if (attr >= 0) { n++; X(attr + '0'); } if (fg >= 0) { if (n) { X(';'); } X('3'); X(fg + '0'); n++; } if (bg >= 0) { if (n) { X(';'); } X('4'); X(bg + '0'); n++; } X('m'); } return out; } std::ostream& Colour::on(std::ostream& s) { s.iword(xindex) = 0; return s; } std::ostream& Colour::off(std::ostream& s) { s.iword(xindex) = 1; return s; } std::ostream& Colour::reset(std::ostream& s) { return put(s, -1, -1, RESET); } std::ostream& Colour::bold(std::ostream& s) { return put(s, -1, -1, BOLD); } std::ostream& Colour::underline(std::ostream& s) { return put(s, -1, -1, UNDERLINE); } std::ostream& Colour::blink(std::ostream& s) { return put(s, -1, -1, BLINK); } std::ostream& Colour::reverse(std::ostream& s) { return put(s, -1, -1, REVERSE); } std::ostream& Colour::hidden(std::ostream& s) { return put(s, -1, -1, HIDDEN); } std::ostream& Colour::black(std::ostream& s) { return put(s, BLACK, -1, -1); } std::ostream& Colour::red(std::ostream& s) { return put(s, RED, -1, -1); } std::ostream& Colour::green(std::ostream& s) { return put(s, GREEN, -1, -1); } std::ostream& Colour::yellow(std::ostream& s) { return put(s, YELLOW, -1, -1); } std::ostream& Colour::magenta(std::ostream& s) { return put(s, MAGENTA, -1, -1); } std::ostream& Colour::blue(std::ostream& s) { return put(s, BLUE, -1, -1); } std::ostream& Colour::cyan(std::ostream& s) { return put(s, CYAN, -1, -1); } std::ostream& Colour::white(std::ostream& s) { return put(s, WHITE, -1, -1); } std::ostream& Colour::blackBackground(std::ostream& s) { return put(s, -1, BLACK, -1); } std::ostream& Colour::redBackground(std::ostream& s) { return put(s, -1, RED, -1); } std::ostream& Colour::greenBackground(std::ostream& s) { return put(s, -1, GREEN, -1); } std::ostream& Colour::yellowBackground(std::ostream& s) { return put(s, -1, YELLOW, -1); } std::ostream& Colour::magentaBackground(std::ostream& s) { return put(s, -1, MAGENTA, -1); } std::ostream& Colour::blueBackground(std::ostream& s) { return put(s, -1, BLUE, -1); } std::ostream& Colour::cyanBackground(std::ostream& s) { return put(s, -1, CYAN, -1); } std::ostream& Colour::whiteBackground(std::ostream& s) { return put(s, -1, WHITE, -1); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/ChannelBuffer.cc0000664000175000017500000000777515161702250020502 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/log/CallbackTarget.h" #include "eckit/log/ChannelBuffer.h" #include "eckit/log/FileTarget.h" #include "eckit/log/IndentTarget.h" #include "eckit/log/LogTarget.h" #include "eckit/log/OStreamTarget.h" #include "eckit/log/TeeTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ChannelBuffer::ChannelBuffer(std::size_t size) : std::streambuf(), target_{nullptr}, buffer_(size) { ASSERT(size); setp(buffer_.data(), buffer_.data() + buffer_.size()); } ChannelBuffer::~ChannelBuffer() { reset(); } bool ChannelBuffer::active() const { return target_ != nullptr; } void ChannelBuffer::setTarget(LogTarget* target) { ASSERT(target); sync(); target->attach(); if (target_) { target_->detach(); } target_ = target; } void ChannelBuffer::addTarget(LogTarget* target) { ASSERT(target); setTarget(new TeeTarget(target_, target)); } void ChannelBuffer::reset() { sync(); if (target_) { target_->detach(); target_ = nullptr; } } bool ChannelBuffer::dumpBuffer() { // When setting and using pointers we should have boundary checks. In a multi threaded environment we already have // experienced weird behaviour. With these checks the race conditions are not gone but they won't cause any // segfaults. See https://github.com/ecmwf/eckit/issues/89 if (target_) { // Explicitly check that `pptr()` is not larger than end of buffer. Racecondition can end up adding larger // values. target_->write(buffer_.data(), std::min(pptr(), buffer_.data() + buffer_.size())); } setp(buffer_.data(), buffer_.data() + buffer_.size()); return true; } void ChannelBuffer::indent(const char* space) { if (target_) { setTarget(new IndentTarget(space, target_)); } } void ChannelBuffer::unindent() { if (target_) { IndentTarget* indent = dynamic_cast(target_); if (indent == nullptr) { throw SeriousBug("Attempt to unindent a Channel that is not indented"); } setTarget(indent->target_); } } void ChannelBuffer::addCallback(channel_callback_t cb, void* data) { setTarget(new TeeTarget(target_, new CallbackTarget(cb, data))); } void ChannelBuffer::setCallback(channel_callback_t cb, void* data) { setTarget(new CallbackTarget(cb, data)); } void ChannelBuffer::addStream(std::ostream& out) { setTarget(new TeeTarget(target_, new OStreamTarget(out))); } void ChannelBuffer::setStream(std::ostream& out) { setTarget(new OStreamTarget(out)); } void ChannelBuffer::addFile(const std::string& path, size_t bufferSize) { setTarget(new TeeTarget(target_, new FileTarget(path, bufferSize))); } void ChannelBuffer::setFile(const std::string& path, size_t bufferSize) { setTarget(new FileTarget(path, bufferSize)); } std::streambuf::int_type ChannelBuffer::overflow(std::streambuf::int_type ch) { if (ch == traits_type::eof()) { return sync(); } dumpBuffer(); return sputc(ch); } std::streambuf::int_type ChannelBuffer::sync() { if (dumpBuffer()) { if (target_) { target_->flush(); } return 0; } return -1; } void ChannelBuffer::print(std::ostream& s) const { s << "ChannelBuffer(size=" << buffer_.size(); if (target_) { s << ", target=" << *target_; } s << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/ResourceUsage.h0000664000175000017500000000430015161702250020373 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File ResourceUsage.h // Baudouin Raoult - ECMWF Oct 16 #ifndef eckit_ResourceUsage_h #define eckit_ResourceUsage_h #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/system/MemoryInfo.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class ResourceUsage { public: explicit ResourceUsage(); /// @param name of the timer, used for output /// @param o output stream to use for output explicit ResourceUsage(const std::string& name, std::ostream& o = Log::info()); /// @param name of the timer, used for output /// @param o output stream to use for output explicit ResourceUsage(const char* name, std::ostream& o = Log::info()); ResourceUsage(const ResourceUsage&) = delete; ResourceUsage& operator=(const ResourceUsage&) = delete; ResourceUsage(ResourceUsage&&) = delete; ResourceUsage& operator=(ResourceUsage&&) = delete; ~ResourceUsage(); protected: // methods void init(); private: // members std::string name_; std::ostream& out_; system::MemoryInfo usage_; std::string hostname_; }; //---------------------------------------------------------------------------------------------------------------------- template class TraceResourceUsage : public ResourceUsage { public: explicit TraceResourceUsage(const char* name) : ResourceUsage(name, eckit::Log::debug()) {} explicit TraceResourceUsage(const std::string& name) : ResourceUsage(name, eckit::Log::debug()) {} }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/FileTarget.h0000664000175000017500000000244315161702250017653 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino #ifndef eckit_log_FileTarget_h #define eckit_log_FileTarget_h #include #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/log/LogTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class FileTarget : public LogTarget { public: explicit FileTarget(const PathName& path, size_t bufferSize = 4 * 1024); virtual ~FileTarget(); private: void write(const char* start, const char* end) override; void flush() override; void print(std::ostream& s) const override; private: std::ofstream out_; Buffer buffer_; //< configurable buffer for ofstream PathName path_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/StatusTarget.cc0000664000175000017500000000166415161702250020421 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/StatusTarget.h" #include "eckit/runtime/Monitor.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- StatusTarget::StatusTarget() {} void StatusTarget::line(const char* line) { Monitor::instance().status(line); } void StatusTarget::print(std::ostream& s) const { s << "StatusTarget()"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/UserChannel.cc0000664000175000017500000000622315161702250020172 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/UserChannel.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class UserBuffer : public std::streambuf { public: using MsgType = UserChannel::MsgType; UserBuffer(std::size_t size = 1024) : std::streambuf(), buffer_(size + 1), msgType_(UserChannel::NONE), user_{nullptr} { ASSERT(size); char* base = &buffer_.front(); setp(base, base + buffer_.size() - 1); // don't consider the space for '\0' } ~UserBuffer() { sync(); } void msgType(MsgType t) { msgType_ = t; } MsgType msgType() const { return msgType_; } void userMsg(UserMsg* p) { user_ = p; } UserMsg* userMsg() const { return user_; } private: std::vector buffer_; ///< internal buffer MsgType msgType_; ///< type of next message UserMsg* user_; ///< interface to user messeges bool dumpBuffer() { std::replace(pbase(), epptr(), '\n', '\0'); switch (msgType_) { case UserChannel::NONE: break; case UserChannel::INFO: if (user_) { user_->infoMsg(pbase()); } break; case UserChannel::WARN: if (user_) { user_->warningMsg(pbase()); } break; case UserChannel::ERROR: if (user_) { user_->errorMsg(pbase()); } break; } setp(pbase(), epptr()); return true; } protected: virtual int_type overflow(int_type ch) { if (ch == traits_type::eof()) { return sync(); } dumpBuffer(); sputc(ch); return traits_type::to_int_type(ch); } virtual int_type sync() { return (dumpBuffer() ? 0 : -1); } }; //---------------------------------------------------------------------------------------------------------------------- UserChannel::UserChannel() : std::ostream(new UserBuffer()), buffer_(dynamic_cast(rdbuf())) { ASSERT(buffer_); } UserChannel::~UserChannel() { delete buffer_; } void UserChannel::msgType(UserChannel::MsgType t) { buffer_->msgType(t); } void UserChannel::userMsg(UserMsg* p) { buffer_->userMsg(p); } UserMsg* UserChannel::userMsg() const { return buffer_->userMsg(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/WrapperTarget.h0000664000175000017500000000246215161702250020415 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file WrapperTarget.h /// @author Baudouin Raoult /// @author Tiago Quintino /// @date August 2016 #ifndef eckit_log_WrapperTarget_h #define eckit_log_WrapperTarget_h #include "eckit/log/LogTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class WrapperTarget : public LogTarget { protected: // methods WrapperTarget(LogTarget* target = nullptr); ~WrapperTarget() override; protected: // members LogTarget* target_; private: void write(const char* start, const char* end) override; void flush() override; virtual void writePrefix() = 0; virtual void writeSuffix() = 0; protected: bool prefix_; friend class ChannelBuffer; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/OStreamTarget.cc0000664000175000017500000000215215161702250020501 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/log/OStreamTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- OStreamTarget::OStreamTarget(std::ostream& out) : out_(out) {} OStreamTarget::~OStreamTarget() {} void OStreamTarget::write(const char* start, const char* end) { if (start >= end) return; out_.write(start, end - start); } void OStreamTarget::flush() { out_.flush(); } void OStreamTarget::print(std::ostream& s) const { s << "OStreamTarget()"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/TCPTarget.h0000664000175000017500000000322415161702250017420 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file TCPTarget.h /// @author Metin Cakircali /// @date May 2025 #include #include #include "eckit/log/LogTarget.h" #include "eckit/net/TCPSocket.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// @brief sends log messages over a TCP class TCPTarget : public LogTarget { public: // methods /// @note takes ownership of [socket] /// @param socket TCP socket to use for log messages explicit TCPTarget(net::TCPSocket& socket); // rules TCPTarget(const TCPTarget&) = delete; TCPTarget& operator=(const TCPTarget&) = delete; TCPTarget(TCPTarget&&) = delete; TCPTarget& operator=(TCPTarget&&) = delete; ~TCPTarget() override = default; void write(const char* start, const char* end) override; void flush() override; protected: // methods void print(std::ostream& out) const override; void write(const std::string& msg); const net::TCPSocket& socket() const { return out_; } private: // members net::TCPSocket out_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/Bytes.h0000664000175000017500000000567715161702250016727 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Jul 1996 #ifndef eckit_log_Bytes_h #define eckit_log_Bytes_h #include "eckit/utils/Literals.h" #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Timer; class Bytes { public: // methods Bytes(double); /// Computes a rate Bytes(double, Timer&); /// Computes a rate Bytes(double, double); operator std::string() const; friend std::ostream& operator<<(std::ostream&, const Bytes&); /// Compute number of bytes in 'n' Kibibytes. /// @note Result will wraparound if n > 2^52-1 /// @param n nuber of Kibibytes static uint64_t KiB(uint64_t n) { using namespace eckit::literals; return 1_KiB * n; } /// Compute number of bytes in 'n' Mebibytes. /// @note Result will wraparound if n > 2^42-1 /// @param n nuber of Mebibytes static uint64_t MiB(uint64_t n) { using namespace eckit::literals; return 1_MiB * n; } /// Compute number of bytes in 'n' Gibibytes. /// @note Result will wraparound if n > 2^32-1 /// @param n nuber of Gibibytes static uint64_t GiB(uint64_t n) { using namespace eckit::literals; return 1_GiB * n; } /// Compute number of bytes in 'n' Tebibytes. /// @note Result will wraparound if n > 2^22-1 /// @param n nuber of Tebibytes static uint64_t TiB(uint64_t n) { using namespace eckit::literals; return 1_TiB * n; } /// Compute number of bytes in 'n' Pebibytes. /// @note Result will wraparound if n > 4095 /// @param n nuber of Pebibytes static uint64_t PiB(uint64_t n) { using namespace eckit::literals; return 1_PiB * n; } /// Compute number of bytes in 'n' Exbibytes. /// @note Result will wraparound if n > 3 /// @param n nuber of Exbibytes static uint64_t EiB(uint64_t n) { using namespace eckit::literals; return 1_EiB * n; } /// Handle rate computations avoiding floating point exceptions static double rate(double num, double den); double value() const; std::string shorten() const; private: // members int sign() const; std::pair reduceTo1024() const; std::pair reduceTo100() const; double bytes_; bool rate_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/JSON.cc0000664000175000017500000001501115161702250016527 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/log/JSON.h" #include "eckit/types/DateTime.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static void print_indent(std::ostream& out, int indent) { indent = std::max(0, indent); out << "\n" << std::string(size_t(indent), ' '); } static bool check(const JSON::Formatting& formatting, int flag) { if (flag == JSON::Formatting::COMPACT) { return formatting.flags() == JSON::Formatting::COMPACT; } return (formatting.flags() & flag) == flag; } JSON::Formatting JSON::Formatting::compact() { return Formatting(COMPACT); } JSON::Formatting JSON::Formatting::indent(int indentation) { return Formatting(INDENT_DICT, indentation); } JSON::Formatting::Formatting(int formatting, int indentation) : flags_{formatting}, indentation_{indentation} {} int JSON::Formatting::indentation() const { return indentation_; } int JSON::Formatting::flags() const { return flags_; } //---------------------------------------------------------------------------------------------------------------------- JSON::JSON(std::ostream& out, bool null) : out_(out), null_(null) { sep_.push_back(""); state_.push_back(true); } JSON::JSON(std::ostream& out, JSON::Formatting formatting) : JSON(out, true) { formatting_ = formatting; } JSON::~JSON() { if (null_) { out_ << "null"; } } void JSON::sep() { null_ = false; out_ << sep_.back(); if (sep_.back() == ",") { bool indent = false; if (check(formatting_, Formatting::INDENT_DICT) && indict()) { indent = true; } if (check(formatting_, Formatting::INDENT_LIST) && inlist()) { indent = true; } if (indent) { print_indent(out_, indentation_); } } std::string colon = check(formatting_, Formatting::COMPACT) ? ":" : " : "; if (indict() && sep_.back() != colon) { sep_.back() = colon; } else { sep_.back() = ","; } } static std::ostream& encode(std::ostream& s, const char* p) { s << '"'; while (*p) { switch (*p) { case '\\': s << "\\\\"; break; case '\n': s << "\\n"; break; case '\t': s << "\\t"; break; case '\b': s << "\\b"; break; case '\f': s << "\\f"; break; case '\r': s << "\\r"; break; case '"': s << "\\\""; break; default: s << *p; break; } p++; } s << '"'; return s; } JSON& JSON::startObject() { null_ = false; sep(); sep_.push_back(""); state_.push_back(true); out_ << "{"; if (check(formatting_, Formatting::INDENT_DICT)) { indentation_ += formatting_.indentation(); print_indent(out_, indentation_); } return *this; } JSON& JSON::null() { null_ = false; sep(); out_ << "null"; return *this; } JSON& JSON::startList() { null_ = false; sep(); sep_.push_back(""); state_.push_back(false); out_ << "["; if (check(formatting_, Formatting::INDENT_LIST)) { indentation_ += formatting_.indentation(); print_indent(out_, indentation_); } return *this; } JSON& JSON::endObject() { sep_.pop_back(); state_.pop_back(); if (check(formatting_, Formatting::INDENT_DICT)) { indentation_ -= formatting_.indentation(); print_indent(out_, indentation_); } out_ << "}"; return *this; } JSON& JSON::endList() { sep_.pop_back(); state_.pop_back(); if (check(formatting_, Formatting::INDENT_LIST)) { indentation_ -= formatting_.indentation(); print_indent(out_, indentation_); } out_ << "]"; return *this; } JSON& JSON::operator<<(bool n) { null_ = false; sep(); out_ << (n ? "true" : "false"); return *this; } JSON& JSON::operator<<(char n) { null_ = false; sep(); out_ << '"' << n << '"'; return *this; } JSON& JSON::operator<<(unsigned char n) { null_ = false; sep(); out_ << '"' << n << '"'; return *this; } JSON& JSON::operator<<(int n) { null_ = false; sep(); out_ << n; return *this; } JSON& JSON::operator<<(unsigned int n) { null_ = false; sep(); out_ << n; return *this; } JSON& JSON::operator<<(long n) { null_ = false; sep(); out_ << n; return *this; } JSON& JSON::operator<<(unsigned long n) { null_ = false; sep(); out_ << n; return *this; } JSON& JSON::operator<<(long long n) { null_ = false; sep(); out_ << n; return *this; } JSON& JSON::operator<<(unsigned long long n) { null_ = false; sep(); out_ << n; return *this; } JSON& JSON::operator<<(float n) { null_ = false; sep(); out_ << n; return *this; } JSON& JSON::operator<<(double n) { null_ = false; sep(); out_ << n; return *this; } JSON& JSON::operator<<(const std::string& s) { null_ = false; sep(); encode(out_, s.c_str()); return *this; } JSON& JSON::operator<<(const char* s) { null_ = false; sep(); encode(out_, s); return *this; } JSON& JSON::operator<<(const Date& date) { *this << std::string(date); return *this; } JSON& JSON::operator<<(const Time& time) { *this << std::string(time); return *this; } JSON& JSON::operator<<(const DateTime& datetime) { *this << datetime.iso(false); return *this; } JSON& JSON::operator<<(const ::timeval& t) { *this << double(t.tv_sec + t.tv_usec / 1000000.0); return *this; } JSON& JSON::precision(int n) { out_ << std::setprecision(n); return *this; } void JSON::raw(const char* buffer, long len) { out_ << std::string(buffer, size_t(len)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/BigNum.cc0000664000175000017500000000223315161702250017141 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/log/BigNum.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- void BigNum::print(std::ostream& s, long long v) { if (v >= 1000) { print(s, v / 1000); s << ','; s << std::setw(3) << std::setfill('0') << (v % 1000); } else { s << v; } } void BigNum::print(std::ostream& s) const { long long v = value_; if (v < 0) { v = -v; s << '-'; } char oldfill = s.fill(); print(s, v); s << std::setfill(oldfill); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/TCPTarget.cc0000664000175000017500000000240215161702250017553 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/TCPTarget.h" #include #include #include "eckit/net/TCPSocket.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- TCPTarget::TCPTarget(net::TCPSocket& socket) : out_(socket) {} void TCPTarget::write(const char* start, const char* end) { if (start && end && end >= start) { out_.write(static_cast(start), end - start); } } void TCPTarget::write(const std::string& msg) { if (!msg.empty()) { out_.write(msg.data(), msg.size()); } } void TCPTarget::flush() { // no-op } void TCPTarget::print(std::ostream& out) const { out << "TCPTarget(" << out_ << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/BigNum.h0000664000175000017500000000213615161702250017005 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File BigNum.h // Baudouin Raoult - ECMWF Nov 96 #ifndef eckit_BigNum_h #define eckit_BigNum_h #include namespace eckit { /// Class used to print large numbers class BigNum { public: BigNum(long long v) : value_(v) {} BigNum(const BigNum&) = delete; BigNum& operator=(const BigNum&) = delete; BigNum(BigNum&&) = delete; BigNum& operator=(BigNum&&) = delete; private: void print(std::ostream&) const; static void print(std::ostream&, long long); private: // members long long value_; friend std::ostream& operator<<(std::ostream& s, const BigNum& p) { p.print(s); return s; } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Seconds.h0000664000175000017500000000254315161702250017224 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Seconds.h // Baudouin Raoult - ECMWF Jul 96 #ifndef eckit_Seconds_h #define eckit_Seconds_h #include #include #include //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- // class Bless; class Seconds { public: // -- Contructors Seconds(double, bool compact = false); Seconds(const struct ::timeval&, bool compact = false); // #include "eckit/types/Seconds.b" // -- Operators operator std::string() const; operator double() const { return seconds_; } friend std::ostream& operator<<(std::ostream&, const Seconds&); private: // There is no private copy constructor as this will confuse g++ 4.x.x // -- Members double seconds_; bool compact_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/TimeStamp.h0000664000175000017500000000176215161702250017533 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Jul 96 #ifndef eckit_log_TimeStamp_h #define eckit_log_TimeStamp_h #include #include #include namespace eckit { class TimeStamp { public: // methods TimeStamp(const std::string& = defaultFormat_); TimeStamp(time_t, const std::string& = defaultFormat_); operator std::string() const; friend std::ostream& operator<<(std::ostream&, const TimeStamp&); private: // members time_t time_; const std::string& format_; static const char* defaultFormat_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/log/SavedStatus.h0000664000175000017500000000156215161702250020074 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file SavedStatus.h /// @author Tiago Quintino #ifndef eckit_log_SavedStatus_h #define eckit_log_SavedStatus_h #include namespace eckit { //----------------------------------------------------------------------------- /// Saves and Restores Monitor status line class SavedStatus { std::string status_; public: SavedStatus(); ~SavedStatus(); }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/JSON.h0000664000175000017500000001116615161702250016400 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Aug 2011 #ifndef eckit_log_JSON_h #define eckit_log_JSON_h #include #include #include #include #include #include #include #include "eckit/eckit.h" namespace eckit { class Time; class Date; class DateTime; //---------------------------------------------------------------------------------------------------------------------- class JSON { public: class Formatting { public: // types enum BitFlags { COMPACT = 0, INDENT_DICT = (1 << 1), INDENT_LIST = (1 << 2), INDENT_ALL = (INDENT_DICT | INDENT_LIST) }; public: // constructors /// Create compact formatting static Formatting compact(); /// Create formatting that indents dictionaries static Formatting indent(int indentation = 2); /// Default constructor, creates compact formatting Formatting() = default; /// Construct formatting with given indentation /// @param flags BitFlags that describe formatting /// @param indentation Number of spaces used for indentation Formatting(int flags, int indentation = 2); public: // methods /// @return Number of spaces used for indentation int indentation() const; /// @return BitFlags that describe formatting int flags() const; private: // data int flags_{COMPACT}; int indentation_{2}; }; public: // methods JSON(std::ostream&, bool null = true); JSON(std::ostream&, Formatting); JSON(const JSON&) = delete; JSON& operator=(const JSON&) = delete; JSON(JSON&&) = delete; JSON& operator=(JSON&&) = delete; ~JSON(); JSON& operator<<(bool); JSON& operator<<(char); JSON& operator<<(unsigned char); JSON& operator<<(int); JSON& operator<<(unsigned int); JSON& operator<<(long); JSON& operator<<(unsigned long); JSON& operator<<(long long); JSON& operator<<(unsigned long long); JSON& operator<<(float); JSON& operator<<(double); JSON& operator<<(const Date&); JSON& operator<<(const Time&); JSON& operator<<(const DateTime&); JSON& operator<<(const ::timeval&); JSON& operator<<(const std::string&); JSON& operator<<(const char*); template JSON& operator<<(const std::vector&); template JSON& operator<<(const std::set&); template JSON& operator<<(const std::map&); JSON& null(); JSON& startObject(); JSON& endObject(); JSON& startList(); JSON& endList(); /// Sets the precision for float and double (works like std::setprecision) JSON& precision(int); /// write raw characters into the json stream /// @warning use with care as this may create invalid json void raw(const char*, long); private: // members std::ostream& out_; std::vector sep_; std::vector state_; bool null_; int indentation_{0}; Formatting formatting_; private: // methods void sep(); bool inlist() { return !state_.back(); } bool indict() { return state_.back(); } }; //---------------------------------------------------------------------------------------------------------------------- template JSON& JSON::operator<<(const std::vector& v) { startList(); for (size_t i = 0; i < v.size(); ++i) { *this << v[i]; } endList(); return *this; } template JSON& JSON::operator<<(const std::set& s) { startList(); for (typename std::set::const_iterator i = s.begin(); i != s.end(); ++i) { *this << *i; } endList(); return *this; } template JSON& JSON::operator<<(const std::map& m) { startObject(); for (typename std::map::const_iterator it = m.begin(); it != m.end(); ++it) { *this << (*it).first << (*it).second; } endObject(); return *this; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/PrefixTarget.cc0000664000175000017500000000245315161702250020370 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/log/PrefixTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- PrefixTarget::PrefixTarget(const std::string& prefix, LogTarget* target, const char* space) : WrapperTarget(target), prefix_(prefix), space_(space), prefixLength_(prefix.size()), spaceLength_(::strlen(space)) {} void PrefixTarget::writePrefix() { const char* p = prefix_.c_str(); target_->write(p, p + prefixLength_); target_->write(space_, space_ + spaceLength_); } void PrefixTarget::writeSuffix() {} void PrefixTarget::print(std::ostream& s) const { s << "PrefixTarget(prefix=" << prefix_ << ", space=" << space_ << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/Log.h0000664000175000017500000001115315161702250016344 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Log.h /// @author Baudouin Raoult /// @author Tiago Quintino /// @date May 1996 #ifndef eckit_log_Log_h #define eckit_log_Log_h #include "eckit/deprecated.h" #include "eckit/log/Channel.h" #include "eckit/log/UserChannel.h" namespace eckit { using channel_callback_t = void (*)(void* data, const char* msg); //---------------------------------------------------------------------------------------------------------------------- /// Singleton holding global streams for logging /// class Log { public: // types /// Output formats enum { compactFormat = 0, normalFormat = 1, fullFormat = 2, monitorFormat = 3, applicationFormat = 4, // Free to use for applications }; public: // methods /// Channel for debug output static Channel& debug(); /// Channel for informative messages static Channel& info(); /// Channel for warning messages static Channel& warning(); /// Channel for error messages static Channel& error(); /// Channel for metrics messages static Channel& metrics(); /// Channel for panic messages static std::ostream& panic(); /// Channel accessible through category index // static Channel& channel(int cat); /// Channel for status messages to Application Monitor static std::ostream& status(); /// Channel for status messages to Application Monitor static std::ostream& message(); /// Get the channel for the user static UserChannel& user(); /// Channel for informative messages tp remote user static std::ostream& userInfo(); /// Channel for warning messages to remote user static std::ostream& userWarning(); /// Channel for error messages to remote user static std::ostream& userError(); /// Send messages to remote user directly -- not using a channel static void notifyClient(const std::string&); /// manipulator that will print the last error message as in perror(2) static std::ostream& syserr(std::ostream&); static Channel& null(); template static Channel& debug(const T* = 0) { return T::instance().debugChannel(); } static void setStream(std::ostream& out); static void addStream(std::ostream& out); static void setFile(const std::string& path); static void addFile(const std::string& path); static void setCallback(channel_callback_t cb, void* data = nullptr); static void addCallback(channel_callback_t cb, void* data = nullptr); static void flush(); static void reset(); static void print(std::ostream& os); private: // methods Log(); ///< Private, non-instanciatable class ~Log(); ///< Private, non-instanciatable class }; //---------------------------------------------------------------------------------------------------------------------- /// Format manipulators int format(std::ostream&); void format(std::ostream&, int); class LogFormatSetter { int format_; public: explicit LogFormatSetter(int f) : format_(f) {} friend std::ostream& operator<<(std::ostream& s, const LogFormatSetter& f) { format(s, f.format_); return s; } }; inline LogFormatSetter setformat(int format) { return LogFormatSetter(format); } // Non-flushing version of std::endl inline std::ostream& newl(std::ostream& out) { return out << '\n'; } //---------------------------------------------------------------------------------------------------------------------- #define ECKIT_DEBUG_HERE std::cerr << " DEBUG () @ " << Here() << std::endl; #define ECKIT_DEBUG_VAR(x) std::cerr << " DEBUG (" << #x << ":" << x << ") @ " << Here() << std::endl; /// Optimisation for DEBUG with elision of stream code. For performance critical code blocks. class Voidify { public: Voidify() {} void operator&(std::ostream&) {} }; #define LOG_DEBUG(condition, lib) \ static_cast(0), !(condition) ? (void)0 : eckit::Voidify() & eckit::Log::debug() #define LOG_DEBUG_LIB(lib) \ static_cast(0), !(lib::instance().debug()) ? (void)0 : eckit::Voidify() & eckit::Log::debug() //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/ProgressTimer.cc0000664000175000017500000000417715161702250020576 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/ProgressTimer.h" #include "eckit/log/ETA.h" #include "eckit/log/Plural.h" #include "eckit/log/Seconds.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ProgressTimer::ProgressTimer(const std::string& name, size_t limit, const std::string& unit, size_t progressCounted, std::ostream& o) : Timer(name, o), limit_(limit), unit_(unit), progressCounted_(progressCounted), progressTimed_(-1), counter_(0), lastTime_(0.) {} ProgressTimer::ProgressTimer(const std::string& name, size_t limit, const std::string& unit, double progressTimed, std::ostream& o) : Timer(name, o), limit_(limit), unit_(unit), progressCounted_(0), progressTimed_(progressTimed), counter_(0), lastTime_(0.) {} ProgressTimer& ProgressTimer::operator++() { hasOutput_ = counter_ && ((progressCounted_ > 0 && counter_ % progressCounted_ == 0) || (progressTimed_ > 0. && lastTime_ + progressTimed_ < elapsed())); if (hasOutput_) { lastTime_ = elapsed(); double rate = counter_ / lastTime_; output() << eckit::Plural(counter_, unit_) << " in " << eckit::Seconds(lastTime_) << ", rate: " << rate << " " << unit_ << "s/s" << ", ETA: " << eckit::ETA((limit_ - counter_) / rate) << std::endl; } if (counter_ < limit_) { ++counter_; } return *this; } eckit::ProgressTimer::operator bool() const { return hasOutput_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/ETA.cc0000664000175000017500000000244415161702250016375 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "ETA.h" #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ETA::ETA(double ETA) : ETA_(ETA) {} ETA::ETA(const ::timeval& time) : ETA_(time.tv_sec + time.tv_usec / 1000000.0) {} std::ostream& operator<<(std::ostream& s, const ETA& sec) { double t = sec.ETA_; long n = t; long hour = n / (60 * 60); n %= (60 * 60); long minutes = n / 60; n %= 60; s << hour << ':' << std::setfill('0') << std::setw(2) << minutes << ':' << std::setfill('0') << std::setw(2) << n << std::setfill(' '); return s; } ETA::operator std::string() const { std::ostringstream s; s << *this; return s.str(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/Seconds.cc0000664000175000017500000000502015161702250017353 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/log/Seconds.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Seconds::Seconds(double seconds, bool compact) : seconds_(seconds), compact_(compact) {} Seconds::Seconds(const ::timeval& time, bool compact) : seconds_(time.tv_sec + time.tv_usec / 1000000.0), compact_(compact) {} static struct { int length_; const char* name_; const char shortName_; } periods[] = { { 7 * 24 * 60 * 60, "week", 'w', }, { 24 * 60 * 60, "day", 'd', }, { 60 * 60, "hour", 'h', }, { 60, "minute", 'm', }, { 1, "second", 's', }, { 0, nullptr, 0, }, }; std::ostream& operator<<(std::ostream& s, const Seconds& sec) { // Use an intermediate std::stringstream, so that the entire // grouping can be manipulated with std::setw(), ... std::ostringstream ss; double t = sec.seconds_; long n = std::lround(t); int flg = 0; for (int i = 0; periods[i].length_; i++) { long m = n / periods[i].length_; if (sec.compact_) { if (m || flg) { ss << m << periods[i].shortName_; n %= periods[i].length_; flg++; } } else { if (m) { if (flg) { ss << ' '; } ss << m << ' ' << periods[i].name_; if (m > 1) { ss << 's'; } n %= periods[i].length_; flg++; } } } if (!flg) { ss << t << (sec.compact_ ? "s" : " second"); } s << ss.str(); return s; } Seconds::operator std::string() const { std::ostringstream s; s << *this; return s.str(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/Bytes.cc0000664000175000017500000000563215161702250017054 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Bytes.h" #include "eckit/log/Timer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// @note supports only until Yottabyte :-) static const char magnitudes[] = {' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'}; Bytes::Bytes(double bytes) : bytes_(bytes), rate_(false) {} Bytes::Bytes(double bytes, Timer& timer) : bytes_(rate(bytes, timer.elapsed())), rate_(true) {} Bytes::Bytes(double bytes, double elapsed) : bytes_(rate(bytes, elapsed)), rate_(true) {} int Bytes::sign() const { return (bytes_ >= 0) ? 1 : -1; } double Bytes::rate(double num, double den) { if (num == 0.) { return 0.; } if (den == 0.) { return num * std::numeric_limits::infinity(); // must be after, num gives sign to inf } return num / den; } double Bytes::value() const { return bytes_; } std::pair Bytes::reduceTo1024() const { double x = std::abs(bytes_); size_t n = 0; while (x >= 1024. && n < NUMBER(magnitudes) - 1) { x /= 1024.; n++; } return std::make_pair(sign() * x, magnitudes[n]); } static const double yotta = 1024. * 1024. * 1024. * 1024. * 1024. * 1024. * 1024. * 1024.; std::pair Bytes::reduceTo100() const { double x = std::abs(bytes_); if (x > yotta || std::isinf(x)) { return std::make_pair(sign() * 99., 'Y'); } size_t n = 0; while (x > 100 && n < NUMBER(magnitudes)) { x /= 1024.; n++; } x = (x >= 10) ? long(x + 0.5) : long(x * 10 + 0.5) / 10.0; return std::make_pair(sign() * x, magnitudes[n]); } std::string Bytes::shorten() const { std::pair r = reduceTo100(); std::ostringstream s; s << r.first << r.second; return s.str(); } std::ostream& operator<<(std::ostream& s, const Bytes& b) { std::pair r = b.reduceTo1024(); s << r.first << ' '; if (r.second != ' ') { s << r.second; } s << "byte"; if (std::abs(r.first) != 1.) { s << 's'; } if (b.rate_) { s << " per second"; } return s; } Bytes::operator std::string() const { std::ostringstream s; s << *this; return s.str(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/LogTarget.cc0000664000175000017500000000154615161702250017656 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/log/LogTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- LogTarget::LogTarget() {} LogTarget::~LogTarget() {} void LogTarget::print(std::ostream& s) const { NOTIMP; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/ResourceUsage.cc0000664000175000017500000000350615161702250020540 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/log/Bytes.h" #include "eckit/log/ResourceUsage.h" #include "eckit/log/Seconds.h" #include "eckit/memory/MMap.h" #include "eckit/memory/Shmget.h" #include "eckit/runtime/Main.h" #include "eckit/system/SystemInfo.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ResourceUsage::ResourceUsage() : name_("unnamed"), out_(std::cout) { init(); } ResourceUsage::ResourceUsage(const std::string& name, std::ostream& o) : name_(name), out_(o) { init(); } ResourceUsage::ResourceUsage(const char* name, std::ostream& o) : name_(name), out_(o) { init(); } void ResourceUsage::init() { using namespace eckit::system; const SystemInfo& sysinfo = SystemInfo::instance(); hostname_ = Main::hostname(); usage_ = sysinfo.memoryUsage(); out_ << "ResourceUsage " << name_ << " => " << hostname_ << " " << usage_ << std::endl; } ResourceUsage::~ResourceUsage() { using namespace eckit::system; const SystemInfo& sysinfo = SystemInfo::instance(); MemoryInfo usage = sysinfo.memoryUsage(); out_ << "ResourceUsage " << name_ << " <= " << hostname_ << " "; usage.delta(out_, usage_); out_ << std::endl; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/MessageTarget.cc0000664000175000017500000000167315161702250020522 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/MessageTarget.h" #include "eckit/runtime/Monitor.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- MessageTarget::MessageTarget() {} void MessageTarget::line(const char* line) { Monitor::instance().message(line); } void MessageTarget::print(std::ostream& s) const { s << "MessageTarget()"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/StatusTarget.h0000664000175000017500000000202015161702250020246 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file StatusTarget.h /// @author Tiago Quintino #ifndef eckit_log_StatusTarget_h #define eckit_log_StatusTarget_h #include "eckit/log/LineBasedTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class StatusTarget : public LineBasedTarget { public: // methods StatusTarget(); private: void line(const char* line) override; void print(std::ostream& s) const override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/log/Channel.cc0000664000175000017500000000443415161702250017335 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/log/Channel.h" #include "eckit/log/ChannelBuffer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Channel::Channel(LogTarget* target) : std::ostream(new ChannelBuffer()), buffer_(dynamic_cast(rdbuf())) { ASSERT(buffer_); if (target) { buffer_->setTarget(target); } // std::cerr << "Channel::Channel()" << std::endl; } Channel::~Channel() { // std::cerr << "Channel::~Channel() " << *this << std::endl; delete buffer_; } bool Channel::operator!() const { bool b = *this; return !b; } Channel::operator bool() const { return buffer_->active(); } void Channel::setCallback(channel_callback_t cb, void* data) { ASSERT(cb); buffer_->setCallback(cb, data); } void Channel::addCallback(channel_callback_t cb, void* data) { ASSERT(cb); buffer_->addCallback(cb, data); } void Channel::setTarget(LogTarget* target) { ASSERT(target); buffer_->setTarget(target); } void Channel::addTarget(LogTarget* target) { ASSERT(target); buffer_->addTarget(target); } void Channel::setStream(std::ostream& out) { buffer_->setStream(out); } void Channel::addStream(std::ostream& out) { buffer_->addStream(out); } void Channel::setFile(const std::string& path) { buffer_->setFile(path); } void Channel::addFile(const std::string& path) { buffer_->addFile(path); } void Channel::reset() { buffer_->reset(); } void Channel::print(std::ostream& s) const { s << "Channel(buffer=" << *buffer_ << ")"; } void Channel::indent(const char* space) { buffer_->indent(space); } void Channel::unindent() { buffer_->unindent(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/log/OStreamTarget.h0000664000175000017500000000220115161702250020336 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file OStreamTarget.h /// @author Tiago Quintino #ifndef eckit_log_OStreamTarget_h #define eckit_log_OStreamTarget_h #include "eckit/log/LogTarget.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class OStreamTarget : public LogTarget { public: // methods OStreamTarget(std::ostream& out); ~OStreamTarget() override; void write(const char* start, const char* end) override; void flush() override; void print(std::ostream& s) const override; private: std::ostream& out_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/config/0000775000175000017500000000000015161702250016135 5ustar alastairalastaireckit-2.0.7/src/eckit/config/Parametrisation.cc0000664000175000017500000000150015161702250021602 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Dec 2018 #include "eckit/config/Parametrisation.h" #include "eckit/exception/Exceptions.h" namespace eckit { Parametrisation::~Parametrisation() {} bool Parametrisation::get(const std::string& name, long long& value) const { NOTIMP; } bool Parametrisation::get(const std::string& name, std::vector& value) const { NOTIMP; } } // namespace eckit eckit-2.0.7/src/eckit/config/Configured.h0000664000175000017500000000540115161702250020373 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #ifndef eckit_Configured_H #define eckit_Configured_H #include #include #include namespace eckit { class Configured { public: // -- Exceptions // None // -- Contructors Configured(); // -- Destructor virtual ~Configured(); // Change to virtual if base class // -- Convertors // None // -- Operators // None // -- Methods virtual Configured& set(const std::string& name, const std::string& value) = 0; virtual Configured& set(const std::string& name, const char* value) = 0; virtual Configured& set(const std::string& name, float value) = 0; virtual Configured& set(const std::string& name, double value) = 0; virtual Configured& set(const std::string& name, int value) = 0; virtual Configured& set(const std::string& name, long value) = 0; // virtual Configured& set(const std::string &name, long long value) = 0; virtual Configured& set(const std::string& name, bool value) = 0; virtual Configured& set(const std::string& name, size_t value) = 0; virtual Configured& set(const std::string& name, const std::vector& value) = 0; virtual Configured& set(const std::string& name, const std::vector& value) = 0; // virtual Configured& set(const std::string& name, const std::vector& value) = 0; virtual Configured& set(const std::string& name, const std::vector& value) = 0; virtual Configured& set(const std::string& name, const std::vector& value) = 0; virtual Configured& set(const std::string& name, const std::vector& value) = 0; virtual Configured& set(const std::string& name, const std::vector& value) = 0; // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // -- Methods // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed // -- Members // None // -- Methods // None // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends }; } // namespace eckit #endif eckit-2.0.7/src/eckit/config/EtcTable.cc0000664000175000017500000000702715161702250020135 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/EtcTable.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/thread/AutoLock.h" #include "eckit/utils/Tokenizer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static const std::vector empty; EtcTable::EtcTable(const std::string& name, int size, const std::string& dir) : last_(0), dir_(dir), name_(name), size_(size) {} EtcTable::~EtcTable() {} const std::vector& EtcTable::lookUp(const std::string& name) { AutoLock lock(mutex_); if (last_ == 0) { load(); } for (std::vector >::const_iterator j = lines_.begin(); j != lines_.end(); ++j) { const std::vector& line = *j; if (match(name, line)) { return line; } } return empty; } std::vector EtcTable::keys() { AutoLock lock(mutex_); if (last_ == 0) { load(); } std::vector v; for (std::vector >::const_iterator j = lines_.begin(); j != lines_.end(); ++j) { const std::vector& line = *j; v.push_back(line[0]); } return v; } std::vector > EtcTable::lines() { AutoLock lock(mutex_); if (last_ == 0) { load(); } return lines_; } bool EtcTable::reload() { AutoLock lock(mutex_); LocalPathName path(std::string("~/") + dir_ + "/" + name_); if (path.lastModified() > last_) { load(); return true; } return false; } bool EtcTable::exists() const { LocalPathName path(std::string("~/") + dir_ + "/" + name_); return path.exists(); } void EtcTable::load() { LocalPathName path(std::string("~/") + dir_ + "/" + name_); std::ifstream in(path.localPath()); last_ = path.lastModified(); Log::info() << "EtcTable::load " << path << std::endl; lines_.clear(); // Log::info() << "Loading table " << path << std::endl; if (!in) { Log::error() << path << Log::syserr << std::endl; return; } char line[1024]; while (in.getline(line, sizeof(line))) { Tokenizer parse(" "); std::vector s; parse(line, s); size_t i = 0; while (i < s.size()) { if (s[i].length() == 0) { s.erase(s.begin() + i); } else { i++; } } if (s.size() == 0 || s[0][0] == '#') { continue; } if (size_ && s.size() != size_) { Log::warning() << "Ignoring " << line << std::endl; } lines_.push_back(s); } } EtcStartWithTable::EtcStartWithTable(const std::string& name, int size, const std::string& dir) : EtcTable(name, size, dir) {} EtcKeyTable::EtcKeyTable(const std::string& name, int size, const std::string& dir) : EtcTable(name, size, dir) {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/config/Configurable.h0000664000175000017500000000363715161702250020717 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Configurable.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_Configurable_h #define eckit_Configurable_h #include #include "eckit/container/ClassExtent.h" #include "eckit/thread/Mutex.h" namespace eckit { //----------------------------------------------------------------------------- class Url; class ResourceBase; class Configurable : public ClassExtent { public: // -- Contructors Configurable(); // -- Destructor virtual ~Configurable(); // -- Class methods static void reconfigureAll(); // Config file as changed, update static void dumpAllResources(std::ostream&); // Dump the configuration to a file static void htmlAllResources(std::ostream&, Url&); /// @returns the name of the class virtual std::string kind() const { return "Configurable"; } /// @returns the name of the instance virtual std::string name() const { return "Unknown"; } protected: // -- Methods virtual void reconfigure() = 0; private: friend class ResourceBase; // -- Members using Set = std::set; Mutex mutex_; Set resources_; // -- Methods void add(ResourceBase*); // Add a resource void remove(ResourceBase*); // Remove a resource void resetResources(); void dumpResources(std::ostream&) const; // Dump all resources to a stream void htmlResources(std::ostream&, Url&); }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/config/YAMLConfiguration.h0000664000175000017500000000323315161702250021601 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date JUl 2015 #ifndef eckit_YAMLConfiguration_H #define eckit_YAMLConfiguration_H #include "eckit/config/Configuration.h" #include "eckit/io/SharedBuffer.h" namespace eckit { class PathName; class Stream; //---------------------------------------------------------------------------------------------------------------------- class YAMLConfiguration : public Configuration { public: YAMLConfiguration(const PathName& path, char separator = '.'); YAMLConfiguration(std::istream&, char separator = '.'); YAMLConfiguration(Stream&, char separator = '.'); YAMLConfiguration(const std::string&, char separator = '.'); YAMLConfiguration(const SharedBuffer&, char separator = '.'); YAMLConfiguration(const YAMLConfiguration&) = delete; YAMLConfiguration& operator=(const YAMLConfiguration&) = delete; YAMLConfiguration(YAMLConfiguration&&) = delete; YAMLConfiguration& operator=(YAMLConfiguration&&) = delete; ~YAMLConfiguration() override; private: // members std::string path_; void print(std::ostream&) const override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/config/EtcTable.h0000664000175000017500000000401515161702250017771 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Aug 2011 #ifndef eckit_EtcTable_h #define eckit_EtcTable_h #include #include #include "eckit/thread/Mutex.h" namespace eckit { class EtcTable { public: // -- Constructors EtcTable(const std::string&, int = 0, const std::string& = "etc"); EtcTable(const EtcTable&) = delete; EtcTable& operator=(const EtcTable&) = delete; EtcTable(EtcTable&&) = delete; EtcTable& operator=(EtcTable&&) = delete; // -- Destructor virtual ~EtcTable(); const std::vector& lookUp(const std::string&); std::vector keys(); std::vector > lines(); bool reload(); bool exists() const; private: // methods void load(); private: // members time_t last_; std::string dir_; std::string name_; size_t size_; Mutex mutex_; std::vector > lines_; private: // methods virtual bool match(const std::string&, const std::vector&) const = 0; }; class EtcKeyTable : public EtcTable { bool match(const std::string& query, const std::vector& line) const { return query == line[0]; } public: EtcKeyTable(const std::string& name, int size = 0, const std::string& dir = "etc"); }; class EtcStartWithTable : public EtcTable { bool match(const std::string& query, const std::vector& line) const { return query.find(line[0]) == 0; } public: EtcStartWithTable(const std::string& name, int size = 0, const std::string& dir = "etc"); }; } // namespace eckit #endif eckit-2.0.7/src/eckit/config/Configurable.cc0000664000175000017500000000361315161702250021047 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/config/Configurable.h" #include "eckit/config/Resource.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Configurable::Configurable() : ClassExtent(this) {} Configurable::~Configurable() {} void Configurable::reconfigureAll() { callAll(&Configurable::resetResources); // Reset all resources callAll(&Configurable::reconfigure); // notify clients } void Configurable::dumpAllResources(std::ostream& s) { callAll(&Configurable::dumpResources, s); } void Configurable::dumpResources(std::ostream& s) const { AutoLock lock(const_cast(*this).mutex_); for (Set::const_iterator i = resources_.begin(); i != resources_.end(); ++i) { (*i)->dump(s); } } // Reset all registered resources void Configurable::resetResources() { AutoLock lock(mutex_); for (Set::iterator i = resources_.begin(); i != resources_.end(); ++i) { (*i)->reset(); } } void Configurable::add(ResourceBase* res) { AutoLock lock(mutex_); resources_.insert(res); } void Configurable::remove(ResourceBase* res) { AutoLock lock(mutex_); resources_.erase(res); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/config/LocalConfiguration.cc0000664000175000017500000001526115161702250022233 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Jul 2015 #include "eckit/config/LocalConfiguration.h" #include "eckit/serialisation/Stream.h" #include "eckit/utils/Tokenizer.h" #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- LocalConfiguration::LocalConfiguration(char separator) : Configuration(Value::makeOrderedMap(), separator) {} LocalConfiguration::LocalConfiguration(Stream& s) : Configuration(Value(s)) {} LocalConfiguration::LocalConfiguration(const Value& root, char separator) : Configuration(root, separator) {} LocalConfiguration::LocalConfiguration(const Configuration& other) : Configuration(other) {} LocalConfiguration::LocalConfiguration(const Configuration& other, const std::string& path) : Configuration(other, path) {} LocalConfiguration::~LocalConfiguration() {} void LocalConfiguration::print(std::ostream& out) const { out << "LocalConfiguration[root="; out << *root_; out << "]"; } void LocalConfiguration::setValue(const std::vector& path, size_t i, eckit::Value& root, const eckit::Value& value) { if (root.shared()) { // std::cout << "Clone " << root << std::endl; root = root.clone(); } if (i + 1 == path.size()) { // std::cout << i << " SET " << path[i] << " to " << value << std::endl; root[path[i]] = value; return; } if (!root.contains(path[i])) { // std::cout << i << " NEW " << path[i] << std::endl; root[path[i]] = eckit::Value::makeOrderedMap(); } eckit::Value& r = root.element(path[i]); setValue(path, i + 1, r, value); } void LocalConfiguration::setValue(const std::string& s, const eckit::Value& value) { // std::cout << "---- " << s << " => " << value << std::endl; eckit::Tokenizer parse(separator_); std::vector path; parse(s, path); setValue(path, 0, *root_, value); } LocalConfiguration& LocalConfiguration::set(const std::string& s, int value) { setValue(s, eckit::Value(value)); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, long value) { setValue(s, eckit::Value(value)); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, long long value) { setValue(s, eckit::Value(value)); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const char* value) { setValue(s, eckit::Value(value)); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const std::string& value) { setValue(s, eckit::Value(value)); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, float value) { setValue(s, eckit::Value(value)); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, double value) { setValue(s, eckit::Value(value)); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, bool value) { setValue(s, eckit::Value(value)); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, size_t value) { setValue(s, eckit::Value(value)); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const std::vector& value) { ValueList values; for (std::vector::const_iterator v = value.begin(); v != value.end(); ++v) { values.push_back(eckit::Value(*v)); } setValue(s, values); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const std::vector& value) { ValueList values; for (std::vector::const_iterator v = value.begin(); v != value.end(); ++v) { values.push_back(eckit::Value(*v)); } setValue(s, values); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const std::vector& value) { ValueList values; for (std::vector::const_iterator v = value.begin(); v != value.end(); ++v) { values.push_back(eckit::Value(*v)); } setValue(s, values); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const std::vector& value) { ValueList values; for (std::vector::const_iterator v = value.begin(); v != value.end(); ++v) { values.push_back(eckit::Value(*v)); } setValue(s, values); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const std::vector& value) { ValueList values; for (std::vector::const_iterator v = value.begin(); v != value.end(); ++v) { values.push_back(eckit::Value(*v)); } setValue(s, values); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const std::vector& value) { ValueList values; for (std::vector::const_iterator v = value.begin(); v != value.end(); ++v) { values.push_back(eckit::Value(*v)); } setValue(s, values); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const std::vector& value) { ValueList values; for (std::vector::const_iterator v = value.begin(); v != value.end(); ++v) { values.push_back(eckit::Value(*v)); } setValue(s, values); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const Configuration& value) { setValue(s, value.getValue()); return *this; } LocalConfiguration& LocalConfiguration::set(const std::string& s, const Configuration* value[], size_t size) { ValueList values; for (size_t i = 0; i < size; ++i) { values.push_back(value[i]->getValue()); } setValue(s, values); return *this; } LocalConfiguration& LocalConfiguration::set(const Configuration& other) { eckit::Value& root = *root_; eckit::Value const& other_root = *other.root_; for (auto& key : other.keys()) { root[key] = other_root[key]; } return *this; } LocalConfiguration& LocalConfiguration::remove(const std::string& name) { root_->remove(name); return *this; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/config/Configured.cc0000664000175000017500000000121215161702250020525 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #include "eckit/config/Configured.h" #include "eckit/exception/Exceptions.h" namespace eckit { Configured::Configured() {} Configured::~Configured() {} } // namespace eckit eckit-2.0.7/src/eckit/config/LibEcKit.cc0000664000175000017500000000364415161702250020101 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date August 2016 #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/eckit_version.h" #include "eckit/thread/AutoLock.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- REGISTER_LIBRARY(LibEcKit); LibEcKit::LibEcKit() : Library("eckit"), abort_handler_(&(::abort)), dontDeregisterFactories_(false) { // can't use Resource here (too early in the initialisation) dontDeregisterFactories_ = (::getenv("ECKIT_DONT_DEREGISTER_FACTORIES") != nullptr); } LibEcKit& LibEcKit::instance() { static LibEcKit libeckit; return libeckit; } void LibEcKit::setAbortHandler(abort_handler_t h) { AutoLock lock(*this); if (h) { abort_handler_ = h; } } void LibEcKit::abort() { abort_handler_(); } bool LibEcKit::dontDeregisterFactories() const { return dontDeregisterFactories_; } const void* LibEcKit::addr() const { return this; } std::string LibEcKit::version() const { return eckit_version_str(); } std::string LibEcKit::gitsha1(unsigned int count) const { std::string sha1(eckit_git_sha1()); if (sha1.empty()) { return "not available"; } return sha1.substr(0, std::min(count, 40u)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/config/JSONConfiguration.h0000664000175000017500000000145715161702250021616 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date JUl 2015 #ifndef eckit_JSONConfiguration_H #define eckit_JSONConfiguration_H #warning eckit::JSONConfiguration is deprecated in favour of eckit::YAMLConfiguration found in header "eckit/config/YAMLConfiguration.h", as a drop-in replacement. #include "eckit/config/YAMLConfiguration.h" namespace eckit { using JSONConfiguration = YAMLConfiguration; } // namespace eckit #endif eckit-2.0.7/src/eckit/config/Configuration.cc0000664000175000017500000004553315161702250021265 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date July 2015 #include "eckit/config/Configuration.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/JSON.h" #include "eckit/utils/Tokenizer.h" #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class ConfigurationNotFound : public Exception { public: ConfigurationNotFound(const std::string& name) { std::ostringstream s; s << "ConfigurationNotFound: [" << name << "]"; reason(s.str()); } }; //---------------------------------------------------------------------------------------------------------------------- Configuration::Configuration(const Configuration& other, const std::string& path) : root_(new Value(*other.root_)), separator_(other.separator_) { bool found = false; *root_ = lookUp(path, found); if (!found) { throw ConfigurationNotFound(path); } } Configuration::Configuration(const Configuration& other) : root_(new Value(*other.root_)), separator_(other.separator_) {} Configuration::Configuration(const eckit::Value& root, char separator) : root_(new Value(root)), separator_(separator) {} Configuration& Configuration::operator=(const Configuration& other) { if (this == &other) { return *this; } *root_ = *other.root_; separator_ = other.separator_; return *this; } Configuration::~Configuration() {} bool Configuration::empty() const { return root_->isNil() || root_->keys().size() == 0; } char Configuration::separator() const { return separator_; } eckit::Value Configuration::lookUp(const std::string& s, bool& found) const { eckit::Tokenizer parse(separator_); std::vector path; parse(s, path); eckit::Value result = *root_; for (size_t i = 0; i < path.size(); i++) { if (!(result.isMap() || result.isOrderedMap())) { found = false; return result; } const std::string& key = path[i]; if (!result.contains(key)) { found = false; return result; } // For some strange reasons clang decide to use // the non-const version, that will clone the internal map const Value& const_result = result; result = const_result[key]; } found = true; return result; } eckit::Configuration::operator Value() const { return *root_; } eckit::Value Configuration::lookUp(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); if (!found) { throw ConfigurationNotFound(name); } return v; } bool Configuration::has(const std::string& name) const { bool found = false; lookUp(name, found); return found; } bool Configuration::get(const std::string& name, std::string& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { value = std::string(v); } return found; } bool Configuration::get(const std::string& name, bool& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { value = v; } return found; } bool Configuration::get(const std::string& name, int& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { long result(v); ASSERT(int(result) == result); value = result; } return found; } bool Configuration::get(const std::string& name, long& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { value = long(v); } return found; } bool Configuration::get(const std::string& name, long long& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { using long_long_t = long long; value = long_long_t(v); } return found; } bool Configuration::get(const std::string& name, size_t& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { value = size_t(v); } return found; } bool Configuration::get(const std::string& name, float& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { value = double(v); } return found; } bool Configuration::get(const std::string& name, double& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { value = v; } return found; } bool Configuration::get(const std::string& name, std::vector& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { ASSERT(v.isList()); value.clear(); int i = 0; while (v.contains(i)) { long result(v[i]); ASSERT(int(result) == result); value.push_back(result); i++; } } return found; } bool Configuration::get(const std::string& name, std::vector& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { ASSERT(v.isList()); value.clear(); int i = 0; while (v.contains(i)) { value.push_back(v[i]); i++; } } return found; } bool Configuration::get(const std::string& name, std::vector& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { ASSERT(v.isList()); value.clear(); int i = 0; while (v.contains(i)) { value.push_back(v[i]); i++; } } return found; } bool Configuration::get(const std::string& name, std::vector& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { ASSERT(v.isList()); value.clear(); int i = 0; while (v.contains(i)) { value.push_back(v[i]); i++; } } return found; } bool Configuration::get(const std::string& name, std::vector& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { ASSERT(v.isList()); value.clear(); int i = 0; while (v.contains(i)) { value.push_back(double(v[i])); i++; } } return found; } bool Configuration::get(const std::string& name, std::vector& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { ASSERT(v.isList()); value.clear(); int i = 0; while (v.contains(i)) { value.push_back(v[i]); i++; } } return found; } bool Configuration::get(const std::string& name, std::vector& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { ASSERT(v.isList()); value.clear(); int i = 0; while (v.contains(i)) { value.push_back(v[i]); i++; } } return found; } bool Configuration::get(const std::string& name, LocalConfiguration& value) const { bool found = has(name); if (found) { value = LocalConfiguration(*this, name); } return found; } const Value& Configuration::getValue() const { return *root_; } void Configuration::hash(Hash& h) const { root_->hash(h); } bool Configuration::get(const std::string& name, std::vector& value) const { bool found = false; eckit::Value v = lookUp(name, found); if (found) { ASSERT(v.isList()); value.clear(); int i = 0; while (v.contains(i)) { value.push_back(LocalConfiguration(v[i], separator_)); i++; } } return found; } //---------------------------------------------------------------------------------------------------------------------- template void Configuration::_get(const std::string& name, T& value) const { if (!get(name, value)) { throw ConfigurationNotFound(name); } } bool Configuration::getBool(const std::string& name) const { bool result; _get(name, result); return result; } int Configuration::getInt(const std::string& name) const { int result; _get(name, result); return result; } long Configuration::getLong(const std::string& name) const { long result; _get(name, result); return result; } size_t Configuration::getUnsigned(const std::string& name) const { size_t result; _get(name, result); return result; } std::int32_t Configuration::getInt32(const std::string& name) const { std::int32_t result; _get(name, result); return result; } std::int64_t Configuration::getInt64(const std::string& name) const { std::int64_t result; _get(name, result); return result; } float Configuration::getFloat(const std::string& name) const { float result; _get(name, result); return result; } double Configuration::getDouble(const std::string& name) const { double result; _get(name, result); return result; } std::string Configuration::getString(const std::string& name) const { std::string result; _get(name, result); return result; } std::vector Configuration::getIntVector(const std::string& name) const { std::vector result; _get(name, result); return result; } std::vector Configuration::getLongVector(const std::string& name) const { std::vector result; _get(name, result); return result; } std::vector Configuration::getUnsignedVector(const std::string& name) const { std::vector result; _get(name, result); return result; } std::vector Configuration::getInt32Vector(const std::string& name) const { std::vector result; _get(name, result); return result; } std::vector Configuration::getInt64Vector(const std::string& name) const { std::vector result; _get(name, result); return result; } std::vector Configuration::getFloatVector(const std::string& name) const { std::vector result; _get(name, result); return result; } std::vector Configuration::getDoubleVector(const std::string& name) const { std::vector result; _get(name, result); return result; } std::vector Configuration::getStringVector(const std::string& name) const { std::vector result; _get(name, result); return result; } std::vector Configuration::getSubConfigurations(const std::string& name) const { std::vector result; _get(name, result); return result; } std::vector Configuration::getSubConfigurations() const { std::vector result; const eckit::Value& v = *root_; int i = 0; while (v.contains(i)) { result.push_back(LocalConfiguration(v[i], separator_)); i++; } return result; } LocalConfiguration Configuration::getSubConfiguration(const std::string& name) const { LocalConfiguration result; if (has(name)) { _get(name, result); } return result; } bool Configuration::isIntegral(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); return found && v.isNumber(); } bool Configuration::isBoolean(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); return found && v.isBool(); } bool Configuration::isFloatingPoint(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); return found && v.isDouble(); } bool Configuration::isString(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); return found && v.isString(); } bool Configuration::isList(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); return found && v.isList(); } bool Configuration::isSubConfiguration(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); return found && (v.isMap() || v.isOrderedMap()); } bool Configuration::isNull(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); return found && (v.isNil()); } bool Configuration::isIntegralList(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); if (found && v.isList()) { if (v.size() == 0) { return true; } auto& firstElement = v[0]; return firstElement.isNumber(); } return false; } bool Configuration::isBooleanList(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); if (found && v.isList()) { if (v.size() == 0) { return true; } auto& firstElement = v[0]; return firstElement.isBool(); } return false; } bool Configuration::isFloatingPointList(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); if (found && v.isList()) { if (v.size() == 0) { return true; } auto& firstElement = v[0]; return firstElement.isDouble(); } return false; } bool Configuration::isStringList(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); if (found && v.isList()) { if (v.size() == 0) { return true; } auto& firstElement = v[0]; return firstElement.isString(); } return false; } bool Configuration::isSubConfigurationList(const std::string& name) const { bool found = false; eckit::Value v = lookUp(name, found); if (found && v.isList()) { if (v.size() == 0) { return true; } auto& firstElement = v[0]; return firstElement.isMap() || firstElement.isOrderedMap(); } return false; } template void Configuration::_getWithDefault(const std::string& name, T& value, const T& defaultVal) const { if (!get(name, value)) { value = defaultVal; } } bool Configuration::getBool(const std::string& name, const bool& defaultVal) const { bool result; _getWithDefault(name, result, defaultVal); return result; } int Configuration::getInt(const std::string& name, const int& defaultVal) const { int result; _getWithDefault(name, result, defaultVal); return result; } long Configuration::getLong(const std::string& name, const long& defaultVal) const { long result; _getWithDefault(name, result, defaultVal); return result; } size_t Configuration::getUnsigned(const std::string& name, const size_t& defaultVal) const { size_t result; _getWithDefault(name, result, defaultVal); return result; } std::int32_t Configuration::getInt32(const std::string& name, const std::int32_t& defaultVal) const { std::int32_t result; _getWithDefault(name, result, defaultVal); return result; } std::int64_t Configuration::getInt64(const std::string& name, const std::int64_t& defaultVal) const { std::int64_t result; _getWithDefault(name, result, defaultVal); return result; } float Configuration::getFloat(const std::string& name, const float& defaultVal) const { float result; _getWithDefault(name, result, defaultVal); return result; } double Configuration::getDouble(const std::string& name, const double& defaultVal) const { double result; _getWithDefault(name, result, defaultVal); return result; } std::string Configuration::getString(const std::string& name, const std::string& defaultVal) const { std::string result; _getWithDefault(name, result, defaultVal); return result; } std::vector Configuration::getIntVector(const std::string& name, const std::vector& defaultValue) const { std::vector result; _getWithDefault(name, result, defaultValue); return result; } std::vector Configuration::getLongVector(const std::string& name, const std::vector& defaultValue) const { std::vector result; _getWithDefault(name, result, defaultValue); return result; } std::vector Configuration::getUnsignedVector(const std::string& name, const std::vector& defaultValue) const { std::vector result; _getWithDefault(name, result, defaultValue); return result; } std::vector Configuration::getInt32Vector(const std::string& name, const std::vector& defaultValue) const { std::vector result; _getWithDefault(name, result, defaultValue); return result; } std::vector Configuration::getInt64Vector(const std::string& name, const std::vector& defaultValue) const { std::vector result; _getWithDefault(name, result, defaultValue); return result; } std::vector Configuration::getFloatVector(const std::string& name, const std::vector& defaultValue) const { std::vector result; _getWithDefault(name, result, defaultValue); return result; } std::vector Configuration::getDoubleVector(const std::string& name, const std::vector& defaultValue) const { std::vector result; _getWithDefault(name, result, defaultValue); return result; } std::vector Configuration::getStringVector(const std::string& name, const std::vector& defaultValue) const { std::vector result; _getWithDefault(name, result, defaultValue); return result; } void Configuration::json(JSON& s) const { s << *root_; } std::vector Configuration::keys() const { std::vector result; eckit::fromValue(result, root_->keys()); return result; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/config/Parametrisation.h0000664000175000017500000000436015161702250021453 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #ifndef eckit_config_Parametrisation_H #define eckit_config_Parametrisation_H #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Parametrisation { public: // methods /// Destructor redundant but fixes sanity compiler warnings virtual ~Parametrisation(); virtual bool has(const std::string& name) const = 0; virtual bool get(const std::string& name, std::string& value) const = 0; virtual bool get(const std::string& name, bool& value) const = 0; virtual bool get(const std::string& name, int& value) const = 0; virtual bool get(const std::string& name, long& value) const = 0; virtual bool get(const std::string& name, long long& value) const; virtual bool get(const std::string& name, size_t& value) const = 0; virtual bool get(const std::string& name, float& value) const = 0; virtual bool get(const std::string& name, double& value) const = 0; virtual bool get(const std::string& name, std::vector& value) const = 0; virtual bool get(const std::string& name, std::vector& value) const = 0; virtual bool get(const std::string& name, std::vector& value) const; virtual bool get(const std::string& name, std::vector& value) const = 0; virtual bool get(const std::string& name, std::vector& value) const = 0; virtual bool get(const std::string& name, std::vector& value) const = 0; virtual bool get(const std::string& name, std::vector& value) const = 0; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/config/Configuration.h0000664000175000017500000002361615161702250021125 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date July 2015 #ifndef eckit_Configuration_H #define eckit_Configuration_H #include #include #include #include "eckit/config/Parametrisation.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class LocalConfiguration; class JSON; class Value; class Hash; class Configuration : public Parametrisation { /// @note Do NOT expose eckit::Value in the interface of configuration /// eckit::Value should remain an internal detail of configuration objects /// Clients should use typed configuration parameters public: // methods // -- Destructor ~Configuration() override; // Fast access, will throw an exception bool getBool(const std::string& name) const; int getInt(const std::string& name) const; long getLong(const std::string& name) const; std::size_t getUnsigned(const std::string& name) const; std::int32_t getInt32(const std::string& name) const; std::int64_t getInt64(const std::string& name) const; float getFloat(const std::string& name) const; double getDouble(const std::string& name) const; std::string getString(const std::string& name) const; std::vector getIntVector(const std::string& name) const; std::vector getLongVector(const std::string& name) const; std::vector getUnsignedVector(const std::string& name) const; std::vector getInt32Vector(const std::string& name) const; std::vector getInt64Vector(const std::string& name) const; std::vector getFloatVector(const std::string& name) const; std::vector getDoubleVector(const std::string& name) const; std::vector getStringVector(const std::string& name) const; // Access with default in case of falure bool getBool(const std::string& name, const bool& defaultValue) const; int getInt(const std::string& name, const int& defaultValue) const; long getLong(const std::string& name, const long& defaultValue) const; std::size_t getUnsigned(const std::string& name, const std::size_t& defaultValue) const; std::int32_t getInt32(const std::string& name, const std::int32_t& defaultValue) const; std::int64_t getInt64(const std::string& name, const std::int64_t& defaultValue) const; float getFloat(const std::string& name, const float& defaultValue) const; double getDouble(const std::string& name, const double& defaultValue) const; std::string getString(const std::string& name, const std::string& defaultValue) const; std::vector getIntVector(const std::string& name, const std::vector& defaultValue) const; std::vector getLongVector(const std::string& name, const std::vector& defaultValue) const; std::vector getUnsignedVector(const std::string& name, const std::vector& defaultValue) const; std::vector getInt32Vector(const std::string& name, const std::vector& defaultValue) const; std::vector getInt64Vector(const std::string& name, const std::vector& defaultValue) const; std::vector getFloatVector(const std::string& name, const std::vector& defaultValue) const; std::vector getDoubleVector(const std::string& name, const std::vector& defaultValue) const; std::vector getStringVector(const std::string& name, const std::vector& defaultValue) const; bool empty() const; std::vector keys() const; // Access to LocalConfiguration std::vector getSubConfigurations(const std::string& name) const; std::vector getSubConfigurations() const; LocalConfiguration getSubConfiguration(const std::string& name) const; char separator() const; // -- Overridden methods bool has(const std::string& name) const override; bool get(const std::string& name, std::string& value) const override; bool get(const std::string& name, bool& value) const override; bool get(const std::string& name, int& value) const override; bool get(const std::string& name, long& value) const override; bool get(const std::string& name, long long& value) const override; bool get(const std::string& name, std::size_t& value) const override; bool get(const std::string& name, float& value) const override; bool get(const std::string& name, double& value) const override; bool get(const std::string& name, std::vector& value) const override; bool get(const std::string& name, std::vector& value) const override; bool get(const std::string& name, std::vector& value) const override; bool get(const std::string& name, std::vector& value) const override; bool get(const std::string& name, std::vector& value) const override; bool get(const std::string& name, std::vector& value) const override; bool get(const std::string& name, std::vector& value) const override; bool get(const std::string& name, std::vector&) const; bool get(const std::string& name, LocalConfiguration&) const; virtual void hash(eckit::Hash&) const; // -- Introspection methods bool isSubConfiguration(const std::string& name) const; bool isIntegral(const std::string& name) const; bool isBoolean(const std::string& name) const; bool isFloatingPoint(const std::string& name) const; bool isString(const std::string& name) const; bool isList(const std::string& name) const; bool isSubConfigurationList(const std::string& name) const; bool isIntegralList(const std::string& name) const; bool isBooleanList(const std::string& name) const; bool isFloatingPointList(const std::string& name) const; bool isStringList(const std::string& name) const; bool isNull(const std::string& name) const; template bool isConvertible(const std::string& name) const { using _T = std::decay_t; if constexpr (std::is_base_of_v) { return isSubConfiguration(name); } else if constexpr (std::is_same_v<_T, int> || std::is_same_v<_T, long> || std::is_same_v<_T, long long> || std::is_same_v<_T, std::size_t>) { return isIntegral(name) || isBoolean(name); } else if constexpr (std::is_same_v<_T, float> || std::is_same_v<_T, double>) { return isFloatingPoint(name) || isIntegral(name) || isBoolean(name); } else if constexpr (std::is_same_v<_T, std::string>) { return isString(name); } else if constexpr (is_vector<_T>::value) { using _V = std::decay_t; if constexpr (std::is_base_of_v) { return isSubConfigurationList(name); } else if constexpr (std::is_same_v<_V, int> || std::is_same_v<_V, long> || std::is_same_v<_V, long long> || std::is_same_v<_V, std::size_t>) { return isIntegralList(name) || isBooleanList(name); } else if constexpr (std::is_same_v<_V, float> || std::is_same_v<_V, double>) { return isFloatingPointList(name) || isIntegralList(name) || isBooleanList(name); } else if constexpr (std::is_same_v<_V, std::string>) { return isStringList(name); } } return false; } template bool isConvertible(const std::string& name, T&) const { return isConvertible(name); } protected: // methods Configuration(const eckit::Value&, char separator = '.'); Configuration(const Configuration&); Configuration(const Configuration&, const std::string& path); ///< sub-select a subconfiguration Configuration& operator=(const Configuration&); Value lookUp(const std::string&) const; Value lookUp(const std::string&, bool&) const; operator Value() const; const Value& getValue() const; protected: // members friend class LocalConfiguration; std::unique_ptr root_; char separator_; private: // methods void json(JSON& s) const; friend JSON& operator<<(JSON& s, const Configuration& v) { v.json(s); return s; } template void _get(const std::string&, T&) const; template void _getWithDefault(const std::string& name, T& value, const T& defaultVal) const; virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream& s, const Configuration& p) { p.print(s); return s; } private: // Helper structs for introspection of template T in isConvertible method template struct is_vector { using type = T; constexpr static bool value = false; }; template struct is_vector> { using type = std::vector; constexpr static bool value = true; }; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/config/ResourceBase.cc0000664000175000017500000000676415161702250021043 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/config/Configurable.h" #include "eckit/config/Resource.h" #include "eckit/config/ResourceMgr.h" #include "eckit/runtime/Main.h" namespace eckit { ResourceBase::ResourceBase(Configurable* owner, const std::string& str) : owner_(owner), inited_(false) { if (owner_) { owner_->add(this); } const char* p = str.c_str(); while (*p) { std::string* s = &name_; char x = *p; int len = 0; switch (x) { case '$': s = &environment_; break; case '-': s = &options_; break; } *s = p; while (*p && *p != ';') { len++; p++; } s->resize(len); if (*p) { p++; } } } ResourceBase::~ResourceBase() { if (owner_) { owner_->remove(this); } } bool ResourceBase::setFromConfigFile() { bool found = false; std::string s; if (owner_) { found = ResourceMgr::lookUp(owner_->kind(), owner_->name(), name_, s); } else { found = ResourceMgr::lookUp("", "", name_, s); } if (found) { setValue(s); } return found; } void ResourceBase::init() { if (inited_) { return; } // First look in config file // First look for an option on the command line if (options_ != "") { for (int i = 1; i < Main::instance().argc(); i++) { if (options_ == Main::instance().argv(i)) { if (i + 1 == Main::instance().argc() || Main::instance().argv(i + 1)[0] == '-') { setValue("true"); } else { setValue(Main::instance().argv(i + 1)); } inited_ = true; return; } } } // Then look for an environment variable if (environment_ != "") { const char* p = ::getenv(environment_.c_str() + 1); if (p) { setValue(p); inited_ = true; return; } } // Otherwise look in the config file if (name_ != "") { if (setFromConfigFile()) { inited_ = true; return; } } // Else use default. This is done in Resource inited_ = true; } std::string ResourceBase::name() const { if (owner_) { return owner_->kind() + '.' + owner_->name() + '.' + name_; } return name_; } void ResourceBase::dump(std::ostream& s) const { // need const_cast here ((ResourceBase*)this)->init(); s << "# " << name_ << ":" << std::endl; if (options_ != "") { s << "# command line option " << options_ << std::endl; } if (environment_ != "") { s << "# environment variable " << environment_ << " "; const char* p = getenv(environment_.c_str() + 1); if (p) { s << "(defined as " << p << ")"; } else { s << "(undefined)"; } s << std::endl; } s << name() << " : " << getValue(); s << std::endl; } } // namespace eckit eckit-2.0.7/src/eckit/config/ResourceMgr.cc0000664000175000017500000001224315161702250020703 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include // for strlen #include #include "eckit/config/ResourceMgr.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/log/Log.h" #include "eckit/runtime/Main.h" #include "eckit/thread/AutoLock.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ResourceMgr& ResourceMgr::instance() { static ResourceMgr theinstance; return theinstance; } bool ResourceMgr::lookUp(const std::string& s1, const std::string& s2, const std::string& s3, std::string& v) { return ResourceMgr::instance().doLookUp(s1, s2, s3, v); } void ResourceMgr::reset() { AutoLock lock(mutex_); resmap_.clear(); inited_ = false; } // This has to be redone static const char* skip_spaces(const char* p) { while (*p && isspace(*p)) { p++; } return p; } bool ResourceMgr::parse(const char* p) { p = skip_spaces(p); if (*p == 0 || *p == '#') { return true; // skip comments } std::string s[3]; int n = 0; for (;;) { const char* q = p; p = skip_spaces(p); while (*p && *p != ':' && *p != '.' && !isspace(*p)) { p++; } int len = p - q; p = skip_spaces(p); s[n] = q; s[n].resize(len); n++; if (n == 3 || *p != '.') { break; } p++; } if (*p != ':') { return false; } switch (n) { case 1: s[2] = s[0]; s[0] = ""; break; case 2: s[2] = s[1]; s[1] = s[0]; s[0] = ""; break; case 3: break; } p++; p = skip_spaces(p); // Remove trailing blanks int l = ::strlen(p) - 1; while (l >= 0 && isspace(p[l])) { l--; } ResourceQualifier x(s[0], s[1], s[2]); std::string t = std::string(p, l + 1); resmap_[x] = t; return true; } void ResourceMgr::readConfigFile(const LocalPathName& file) { // Log::info() << "ResourceMgr::readConfigFile(" << file << ")" << std::endl; int cnt = 0; std::ifstream in(file.localPath()); char line[1024]; while (in.getline(line, sizeof(line))) { cnt++; if (!parse(line)) { Log::warning() << "Invalid line, file " << file << " line " << cnt << " = " << line << std::endl; } } } void ResourceMgr::set(const std::string& name, const std::string& value) { AutoLock lock(mutex_); std::string s = name + ": " + value; if (!parse(s.c_str())) { Log::warning() << "Failed to parse " << s << std::endl; } } bool ResourceMgr::doLookUp(const std::string& kind, const std::string& owner, const std::string& name, std::string& result) { AutoLock lock(mutex_); if (!inited_) { inited_ = true; readConfigFile("~/etc/config/general"); readConfigFile("~/etc/config/local"); readConfigFile(std::string("~/etc/config/") + Main::instance().name()); readConfigFile(std::string("~/etc/config/") + Main::instance().name() + ".local"); } ResMap::iterator i = resmap_.find(ResourceQualifier(kind, owner, name)); if (i != resmap_.end()) { result = (*i).second; return true; } i = resmap_.find(ResourceQualifier("", owner, name)); if (i != resmap_.end()) { result = (*i).second; return true; } i = resmap_.find(ResourceQualifier("", "", name)); if (i != resmap_.end()) { result = (*i).second; return true; } return false; } bool ResourceMgr::registCmdArgOptions(const std::string&) { AutoLock lock(mutex_); NOTIMP; } ResourceMgr::ResourceMgr() : resmap_(), resoptions_(), inited_(false) {} //---------------------------------------------------------------------------------------------------------------------- ResourceQualifier::ResourceQualifier(const std::string& kind, const std::string& owner, const std::string& name) : kind_(kind), owner_(owner), name_(name) {} ResourceQualifier::ResourceQualifier(const ResourceQualifier& other) { kind_ = other.kind_; owner_ = other.owner_; name_ = other.name_; } int ResourceQualifier::operator<(const ResourceQualifier& other) const { // to be rewritten char buf1[1024]; char buf2[1024]; snprintf(buf1, sizeof(buf1), "%s.%s.%s", owner_.c_str(), kind_.c_str(), name_.c_str()); snprintf(buf2, sizeof(buf2), "%s.%s.%s", other.owner_.c_str(), other.kind_.c_str(), other.name_.c_str()); return strcmp(buf1, buf2) < 0; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/config/YAMLConfiguration.cc0000664000175000017500000000446415161702250021746 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Jul 2015 #include #include "eckit/config/LibEcKit.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/log/Log.h" #include "eckit/parser/YAMLParser.h" #include "eckit/value/Value.h" namespace eckit { static Value root(std::istream& in) { ASSERT(in); eckit::YAMLParser parser(in); Value root = parser.parse(); return root; } static Value root(const std::string& path) { LOG_DEBUG_LIB(LibEcKit) << "Reading YAMLConfiguration from file " << path << std::endl; std::ifstream in(path.c_str()); if (!in) { throw eckit::CantOpenFile(path); } return root(in); } static Value root(Stream& in) { std::string val; in.next(val); std::istringstream iss(val); return root(iss); } static Value root_from_string(const std::string& str) { LOG_DEBUG_LIB(LibEcKit) << "Reading YAMLConfiguration from string:" << std::endl; LOG_DEBUG_LIB(LibEcKit) << str << std::endl; std::istringstream in(str); return root(in); } YAMLConfiguration::YAMLConfiguration(const PathName& path, char separator) : Configuration(root(path), separator), path_(path) {} YAMLConfiguration::YAMLConfiguration(std::istream& in, char separator) : Configuration(root(in), separator), path_("") {} YAMLConfiguration::YAMLConfiguration(Stream& in, char separator) : Configuration(root(in), separator), path_("") {} YAMLConfiguration::YAMLConfiguration(const std::string& str, char separator) : Configuration(root_from_string(str), separator), path_("") {} YAMLConfiguration::YAMLConfiguration(const SharedBuffer& buffer, char separator) : Configuration(root_from_string(buffer.str()), separator), path_("") {} YAMLConfiguration::~YAMLConfiguration() {} void YAMLConfiguration::print(std::ostream& out) const { out << "YAMLConfiguration[path=" << path_ << ", root=" << *root_ << "]"; } } // namespace eckit eckit-2.0.7/src/eckit/config/ResourceMgr.h0000664000175000017500000000513215161702250020544 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date May 96 #ifndef eckit_config_ResourceMgr_H #define eckit_config_ResourceMgr_H #include #include #include "eckit/thread/Mutex.h" namespace eckit { class LocalPathName; //---------------------------------------------------------------------------------------------------------------------- // A resource specifier class ResourceQualifier { std::string kind_; // Kind, e.g. "Application" std::string owner_; // Owner, e.g. "mars" std::string name_; // Name, e.g. "debug" public: ResourceQualifier(); ResourceQualifier(const std::string&, const std::string&, const std::string&); ResourceQualifier(const ResourceQualifier&); ResourceQualifier& operator=(const ResourceQualifier&); int operator<(const ResourceQualifier&) const; }; //---------------------------------------------------------------------------------------------------------------------- class ResourceMgr { public: // class methods static ResourceMgr& instance(); static bool lookUp(const std::string&, const std::string&, const std::string&, std::string&); bool registCmdArgOptions(const std::string&); private: ResourceMgr(); ResourceMgr(const ResourceMgr&) = delete; ResourceMgr& operator=(const ResourceMgr&) = delete; ResourceMgr(ResourceMgr&&) = delete; ResourceMgr& operator=(ResourceMgr&&) = delete; bool doLookUp(const std::string&, const std::string&, const std::string&, std::string&); // Only for my friends // You should never call these, resources have a read-only semantics void reset(); void set(const std::string&, const std::string&); friend class ConfigCmd; friend class ResourceBase; private: // methods void readConfigFile(const LocalPathName&); bool parse(const char*); private: // members using ResMap = std::map; ResMap resmap_; std::map resoptions_; Mutex mutex_; bool inited_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/config/ResourceBase.h0000664000175000017500000000361115161702250020671 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date May 1996 #ifndef eckit_config_ResourceBase_h #define eckit_config_ResourceBase_h #include #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Configurable; class Url; class ResourceBase { public: // methods ResourceBase(Configurable* owner, const std::string& str); ResourceBase(const ResourceBase&) = delete; ResourceBase& operator=(const ResourceBase&) = delete; ResourceBase(ResourceBase&&) = delete; ResourceBase& operator=(ResourceBase&&) = delete; virtual ~ResourceBase(); void reset() { inited_ = false; } void dump(std::ostream&) const; void GET(std::ostream&, Url&); std::string name() const; protected: // methods void init(); virtual bool setFromConfigFile(); private: // members Configurable* owner_; std::string name_; // In the config file std::string environment_; // In the environment variables std::string options_; // For the command line options bool inited_; private: // methods virtual void setValue(const std::string&) = 0; virtual std::string getValue() const = 0; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/config/LibEcKit.h0000664000175000017500000000246015161702250017736 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date August 2016 #pragma once #include "eckit/system/Library.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- using abort_handler_t = void (*)(); class LibEcKit : public eckit::system::Library { public: // methods LibEcKit(); static LibEcKit& instance(); void setAbortHandler(abort_handler_t h); void abort(); bool dontDeregisterFactories() const; protected: // methods const void* addr() const; virtual std::string version() const; virtual std::string gitsha1(unsigned int count) const; private: // members abort_handler_t abort_handler_; bool dontDeregisterFactories_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/config/Resource.h0000664000175000017500000000567215161702250020107 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date May 1996 #pragma once #include #include "eckit/config/Configuration.h" #include "eckit/config/ResourceBase.h" #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template class Resource : public ResourceBase { public: // methods // Standalone Resource(const std::string& str, const T& value) : ResourceBase(nullptr, str), value_(value) {} // Part of a configurable Resource(Configurable* owner, const std::string& str, const T& value) : ResourceBase(owner, str), value_(value) {} Resource(const std::string& str, const std::string& value, bool) : ResourceBase(nullptr, str), value_(eckit::Translator()(value)) {} // -- Convertors operator const T&() const { const_cast*>(this)->init(); return value_; } private: // members T value_; // From ResourceBase virtual void setValue(const std::string& s) { value_ = Translator()(s); } virtual std::string getValue() const { return Translator()(value_); } }; template std::ostream& operator<<(std::ostream& os, const Resource& r) { os << static_cast(r); return os; } //---------------------------------------------------------------------------------------------------------------------- template class LibResource : public ResourceBase { public: // methods LibResource(const std::string& str, const T& value) : ResourceBase(nullptr, str), value_(value) {} operator const T&() const { const_cast*>(this)->init(); return value_; } friend std::ostream& operator<<(std::ostream& os, const LibResource& r) { os << static_cast(r); return os; } private: // members T value_; // From ResourceBase virtual void setValue(const std::string& s) { value_ = Translator()(s); } virtual std::string getValue() const { return Translator()(value_); } virtual bool setFromConfigFile() { std::string value; if (LIB::instance().configuration().get(name(), value)) { setValue(value); return true; } return false; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/config/LocalConfiguration.h0000664000175000017500000001130115161702250022064 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date July 2015 #ifndef eckit_LocalConfiguration_H #define eckit_LocalConfiguration_H #include #include #include #include "eckit/config/Configuration.h" #include "eckit/config/Configured.h" #include "eckit/types/Types.h" namespace eckit { class PathName; class Stream; //---------------------------------------------------------------------------------------------------------------------- class LocalConfiguration : public Configuration, public Configured { /// @note Do NOT expose eckit::Value in the interface of configuration /// eckit::Value should remain an internal detail of configuration objects public: // methods explicit LocalConfiguration(char separator = '.'); explicit LocalConfiguration(Stream& s); explicit LocalConfiguration(const Configuration& other); LocalConfiguration(const Configuration& other, const std::string& path); ~LocalConfiguration() override; LocalConfiguration& set(const Configuration& other); LocalConfiguration& set(const std::string& name, const std::string& value) override; LocalConfiguration& set(const std::string& name, const char* value) override; LocalConfiguration& set(const std::string& name, bool value) override; LocalConfiguration& set(const std::string& name, int value) override; LocalConfiguration& set(const std::string& name, long value) override; LocalConfiguration& set(const std::string& name, long long value); LocalConfiguration& set(const std::string& name, size_t value) override; LocalConfiguration& set(const std::string& name, float value) override; LocalConfiguration& set(const std::string& name, double value) override; LocalConfiguration& set(const std::string& name, const std::vector& value) override; LocalConfiguration& set(const std::string& name, const std::vector& value) override; LocalConfiguration& set(const std::string& name, const std::vector& value); LocalConfiguration& set(const std::string& name, const std::vector& value) override; LocalConfiguration& set(const std::string& name, const std::vector& value) override; LocalConfiguration& set(const std::string& name, const std::vector& value) override; LocalConfiguration& set(const std::string& name, const std::vector& value) override; LocalConfiguration& set(const std::string& name, const Configuration& value); template >>> LocalConfiguration& set(const std::string& name, const std::vector& value) { std::vector abstract_pointers; abstract_pointers.reserve(value.size()); for (auto& v : value) { abstract_pointers.emplace_back(&v); } return set(name, abstract_pointers.data(), abstract_pointers.size()); } template >>> LocalConfiguration& set(const std::string& name, std::initializer_list&& value) { std::vector abstract_pointers; abstract_pointers.reserve(value.size()); for (auto& v : value) { abstract_pointers.emplace_back(&v); } return set(name, abstract_pointers.data(), abstract_pointers.size()); } LocalConfiguration& remove(const std::string& name); protected: friend class Configuration; /// to be used only by class Configuration LocalConfiguration(const eckit::Value&, char separator = '.'); void print(std::ostream&) const override; LocalConfiguration& set(const std::string& name, const Configuration* abstract_pointers[], size_t size); private: void setValue(const std::vector& path, size_t i, Value& root, const Value& value); void setValue(const std::string& s, const Value& value); }; //---------------------------------------------------------------------------------------------------------------------- template <> struct VectorPrintSelector { using selector = VectorPrintSimple; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/eckit_config.h.in0000664000175000017500000000326615161702250020101 0ustar alastairalastair#ifndef eckit_config_h #define eckit_config_h #include "eckit/eckit_ecbuild_config.h" #include "eckit/eckit_version.h" // endiness #cmakedefine01 eckit_BIG_ENDIAN #cmakedefine01 eckit_LITTLE_ENDIAN // system #define ECKIT_SIZEOF_TIME_T @ECKIT_SIZEOF_TIME_T@ #cmakedefine01 eckit_HAVE_DLFCN_H #cmakedefine01 eckit_HAVE_DLADDR #cmakedefine01 eckit_HAVE_MAP_ANONYMOUS #cmakedefine01 eckit_HAVE_MAP_ANON #cmakedefine01 eckit_HAVE_FUNOPEN #cmakedefine01 eckit_HAVE_FSYNC #cmakedefine01 eckit_HAVE_FDATASYNC #cmakedefine01 eckit_HAVE_F_FULLFSYNC #cmakedefine01 eckit_HAVE_FMEMOPEN #cmakedefine01 eckit_HAVE_DLINFO #cmakedefine01 eckit_HAVE_FOPENCOOKIE #cmakedefine01 eckit_HAVE_EXECINFO_BACKTRACE #cmakedefine01 eckit_HAVE_CXXABI_H #cmakedefine01 eckit_HAVE_GMTIME_R #cmakedefine01 eckit_HAVE_READDIR_R #cmakedefine01 eckit_HAVE_DIRFD #cmakedefine01 eckit_HAVE_DIRENT_D_TYPE #cmakedefine01 eckit_HAVE_CXX_INT_128 #cmakedefine01 eckit_HAVE_AIO #cmakedefine01 eckit_HAVE_UNICODE #cmakedefine01 eckit_HAVE_XXHASH // maths #cmakedefine01 eckit_HAVE_FEENABLEEXCEPT #cmakedefine01 eckit_HAVE_FEDISABLEEXCEPT // memory #cmakedefine01 eckit_HAVE_ECKIT_MEMORY_FACTORY_BUILDERS_DEBUG #cmakedefine01 eckit_HAVE_ECKIT_MEMORY_FACTORY_EMPTY_DESTRUCTION // external packages #cmakedefine01 eckit_HAVE_CUDA #cmakedefine01 eckit_HAVE_CURL #cmakedefine01 eckit_HAVE_EIGEN #cmakedefine01 eckit_HAVE_LAPACK #cmakedefine01 eckit_HAVE_MKL #cmakedefine01 eckit_HAVE_MPI #cmakedefine01 eckit_HAVE_OMP #cmakedefine01 eckit_HAVE_SSL #cmakedefine01 eckit_HAVE_ZIP // Have we built certain libraries #cmakedefine01 eckit_HAVE_ECKIT_CMD #cmakedefine01 eckit_HAVE_ECKIT_SQL #cmakedefine01 eckit_HAVE_ECKIT_MPI #endif // eckit_config_h eckit-2.0.7/src/eckit/geometry/0000775000175000017500000000000015161702250016523 5ustar alastairalastaireckit-2.0.7/src/eckit/geometry/Sphere.h0000664000175000017500000000575715161702250020140 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef Sphere_H #define Sphere_H //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- class Point2; class Point3; //---------------------------------------------------------------------------------------------------------------------- struct Sphere { /// Great-circle central angle between two points (latitude/longitude coordinates) in radians static double centralAngle(const Point2& Alonlat, const Point2& Blonlat, bool normalise_angle = false); /// Great-circle central angle between two points (Cartesian coordinates) in radians static double centralAngle(const double& radius, const Point3& A, const Point3& B); /// Great-circle distance between two points (latitude/longitude coordinates) in metres static double distance(const double& radius, const Point2& Alonlat, const Point2& Blonlat); /// Great-circle distance between two points (Cartesian coordinates) in metres static double distance(const double& radius, const Point3& A, const Point3& B); /// Surface area in square metres static double area(const double& radius); /// Surface area between parallels and meridians defined by two points (longitude/latitude /// coordinates) in square metres static double area(const double& radius, const Point2& Alonlat, const Point2& Blonlat); // Great-circle intermediate latitude provided two circle points (A, B) and intermediate // longitude (C) in degrees static double greatCircleLatitudeGivenLongitude(const Point2& Alonlat, const Point2& Blonlat, const double& Clon); // Great-circle intermediate longitude(s) provided two circle points (A, B) and intermediate // latitude (C) in degrees static void greatCircleLongitudeGivenLatitude(const Point2& Alonlat, const Point2& Blonlat, const double& Clat, double& Clon1, double& Clon2); // Convert spherical coordinates to Cartesian static void convertSphericalToCartesian(const double& radius, const Point2& Alonlat, Point3& B, double height = 0., bool normalise_angle = false); // Convert Cartesian coordinates to spherical static void convertCartesianToSpherical(const double& radius, const Point3& A, Point2& Blonlat); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry #endif eckit-2.0.7/src/eckit/geometry/polygon/0000775000175000017500000000000015161702250020212 5ustar alastairalastaireckit-2.0.7/src/eckit/geometry/polygon/Polygon.h0000664000175000017500000000305715161702250022017 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geometry/Point2.h" //------------------------------------------------------------------------------------------------------ namespace eckit::geometry::polygon { //------------------------------------------------------------------------------------------------------ class Polygon : protected std::deque { public: using container_type = std::deque; using container_type::value_type; Polygon() = default; Polygon(std::initializer_list l) : container_type(l) {} using container_type::push_back; using container_type::push_front; size_t num_vertices() const { return size(); } const value_type& vertex(size_t idx) const { return at(idx); } bool sameAs(const Polygon& p) const { return *this == p; } bool congruent(const Polygon&) const; void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const Polygon& p) { p.print(s); return s; } }; //------------------------------------------------------------------------------------------------------ } // namespace eckit::geometry::polygon eckit-2.0.7/src/eckit/geometry/polygon/Polygon.cc0000664000175000017500000000314515161702250022153 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/geometry/polygon/Polygon.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry::polygon { //---------------------------------------------------------------------------------------------------------------------- bool Polygon::congruent(const Polygon& p) const { if (empty()) { return true; } if (size() != p.size()) { return false; } int offset = -1; for (int i = 0; i < size(); i++) { if (at(i) == p.at(0)) { offset = i; break; } } if (offset == -1) { return false; } for (int i = 1; i < size(); i++) { if (at((i + offset) % size()) != p.at(i)) { return false; } } return true; } void Polygon::print(std::ostream& s) const { if (empty()) { s << "[]"; return; } char z = '['; for (const auto& v : *this) { s << z << v; z = ','; } s << ']'; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry::polygon eckit-2.0.7/src/eckit/geometry/polygon/LonLatPolygon.cc0000664000175000017500000001320415161702250023262 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geometry/polygon/LonLatPolygon.h" #include #include #include "eckit/exception/Exceptions.h" #include "eckit/geometry/CoordinateHelpers.h" #include "eckit/types/FloatCompare.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry::polygon { //---------------------------------------------------------------------------------------------------------------------- namespace { inline bool is_approximately_equal(double a, double b) { return types::is_approximately_equal(a, b, 1e-10); } inline bool is_approximately_greater_or_equal(double a, double b) { return a >= b || is_approximately_equal(a, b); } inline double cross_product_analog(const Point2& A, const Point2& B, const Point2& C) { return (A.x() - C.x()) * (B.y() - C.y()) - (A.y() - C.y()) * (B.x() - C.x()); } inline int on_direction(double a, double b, double c) { return a <= b && b <= c ? 1 : c <= b && b <= a ? -1 : 0; }; inline int on_side(const Point2& P, const Point2& A, const Point2& B) { const auto p = cross_product_analog(P, A, B); return is_approximately_equal(p, 0) ? 0 : p > 0 ? 1 : -1; } } // namespace //---------------------------------------------------------------------------------------------------------------------- LonLatPolygon::LonLatPolygon(const std::vector& points, bool includePoles) : container_type(points) { ASSERT(points.size() > 1); ASSERT(is_approximately_equal(points.front()[LON], points.back()[LON]) && is_approximately_equal(points.front()[LAT], points.back()[LAT])); if (points.size() > 2) { clear(); // assumes reserved size is kept push_back(points.front()); push_back(points[1]); for (size_t i = 2; i < points.size(); ++i) { auto A = points[i]; // if new point is aligned with existing edge (cross product ~= 0) make the edge longer const auto& B = back(); const auto& C = operator[](size() - 2); if (is_approximately_equal(0., cross_product_analog(A, B, C))) { back() = A; continue; } push_back(A); } } max_ = min_ = front(); for (const auto& p : *this) { min_ = value_type::componentsMin(min_, p); max_ = value_type::componentsMax(max_, p); } includeNorthPole_ = includePoles && is_approximately_equal(max_[LAT], 90); includeSouthPole_ = includePoles && is_approximately_equal(min_[LAT], -90); ASSERT(is_approximately_greater_or_equal(min_[LAT], -90)); ASSERT(is_approximately_greater_or_equal(90, max_[LAT])); quickCheckLongitude_ = is_approximately_greater_or_equal(360, max_[LON] - min_[LON]); } void LonLatPolygon::print(std::ostream& out) const { out << "["; const auto* sep = ""; for (const auto& p : *this) { out << sep << p; sep = ", "; } out << "]"; } std::ostream& operator<<(std::ostream& out, const LonLatPolygon& pc) { pc.print(out); return out; } bool LonLatPolygon::contains(const Point2& Plonlat, bool normalise_angle) const { if (!normalise_angle) { assert_latitude_range(Plonlat[LAT]); } const Point2 p = canonicaliseOnSphere(Plonlat, min_[LON]); auto lat = p[LAT]; auto lon = p[LON]; // check poles if (includeNorthPole_ && is_approximately_equal(lat, 90)) { return true; } if (includeSouthPole_ && is_approximately_equal(lat, -90)) { return true; } // check bounding box if (!is_approximately_greater_or_equal(lat, min_[LAT]) || !is_approximately_greater_or_equal(max_[LAT], lat)) { return false; } if (quickCheckLongitude_) { if (!is_approximately_greater_or_equal(lon, min_[LON]) || !is_approximately_greater_or_equal(max_[LON], lon)) { return false; } } do { // winding number int wn = 0; int prev = 0; // loop on polygon edges for (size_t i = 1; i < size(); ++i) { const auto& A = operator[](i - 1); const auto& B = operator[](i); // check point-edge side and direction, testing if P is on|above|below (in latitude) of a A,B polygon edge, // by: // - intersecting "up" on forward crossing & P above edge, or // - intersecting "down" on backward crossing & P below edge const auto direction = on_direction(A[LAT], lat, B[LAT]); if (direction != 0) { const auto side = on_side({lon, lat}, A, B); if (side == 0 && on_direction(A[LON], lon, B[LON]) != 0) { return true; } if ((prev != 1 && direction > 0 && side > 0) || (prev != -1 && direction < 0 && side < 0)) { prev = direction; wn += direction; } } } // wn == 0 only when P is outside if (wn != 0) { return true; } lon += 360; } while (lon <= max_[LON]); return false; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry::polygon eckit-2.0.7/src/eckit/geometry/polygon/LonLatPolygon.h0000664000175000017500000000501715161702250023127 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geometry/Point2.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry::polygon { //---------------------------------------------------------------------------------------------------------------------- class LonLatPolygon : protected std::vector { public: // -- Types using container_type = std::vector; using container_type::value_type; // -- Constructors explicit LonLatPolygon(const container_type& points, bool includePoles = true); template LonLatPolygon(Point2Iterator begin, Point2Iterator end, bool includePoles = true) : LonLatPolygon(container_type(begin, end), includePoles) {} LonLatPolygon(const LonLatPolygon&) = default; LonLatPolygon(LonLatPolygon&&) = default; // -- Destructor virtual ~LonLatPolygon() = default; // -- Operators LonLatPolygon& operator=(const LonLatPolygon&) = default; LonLatPolygon& operator=(LonLatPolygon&&) = default; // -- Methods const Point2& max() const { return max_; } const Point2& min() const { return min_; } using container_type::operator[]; using container_type::size; /// @brief Point-in-polygon test based on winding number /// @note reference Inclusion of a Point in a Polygon /// @param[in] P given point /// @param[in] normalise_angle normalise point angles /// @return if point (lon,lat) is in polygon bool contains(const Point2& Plonlat, bool normalise_angle = false) const; private: // -- Methods void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream&, const LonLatPolygon&); // -- Members Point2 max_; Point2 min_; bool includeNorthPole_; bool includeSouthPole_; bool quickCheckLongitude_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry::polygon eckit-2.0.7/src/eckit/geometry/EllipsoidOfRevolution.cc0000664000175000017500000000477615161702250023350 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geometry/EllipsoidOfRevolution.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/geometry/CoordinateHelpers.h" #include "eckit/geometry/Point2.h" #include "eckit/geometry/Point3.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- namespace { static const double degrees_to_radians = M_PI / 180.; } // namespace //---------------------------------------------------------------------------------------------------------------------- void EllipsoidOfRevolution::convertSphericalToCartesian(const double& a, const double& b, const Point2& Alonlat, Point3& B, double height, bool normalise_angle) { ASSERT(a > 0.); ASSERT(b > 0.); // See https://en.wikipedia.org/wiki/Reference_ellipsoid#Coordinates // numerical conditioning for both ϕ (poles) and λ (Greenwich/Date Line) if (!normalise_angle) { assert_latitude_range(Alonlat[1]); } const Point2 alonlat = canonicaliseOnSphere(Alonlat, -180.); const double lambda_deg = alonlat[0]; const double lambda = degrees_to_radians * lambda_deg; const double phi = degrees_to_radians * alonlat[1]; const double sin_phi = std::sin(phi); const double cos_phi = std::sqrt(1. - sin_phi * sin_phi); const double sin_lambda = std::abs(lambda_deg) < 180. ? std::sin(lambda) : 0.; const double cos_lambda = std::abs(lambda_deg) > 90. ? std::cos(lambda) : std::sqrt(1. - sin_lambda * sin_lambda); const double N_phi = a * a / std::sqrt(a * a * cos_phi * cos_phi + b * b * sin_phi * sin_phi); B[0] = (N_phi + height) * cos_phi * cos_lambda; B[1] = (N_phi + height) * cos_phi * sin_lambda; B[2] = (N_phi * (b * b) / (a * a) + height) * sin_phi; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry eckit-2.0.7/src/eckit/geometry/SphereT.h0000664000175000017500000000744015161702250020253 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SphereT_H #define SphereT_H #include "eckit/geometry/Sphere.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- /// Definition of a sphere parametrised with a geodetic datum template struct SphereT { /// Sphere radius in metres inline static double radius() { return DATUM::radius(); } /// Great-circle central angle between two points (longitude/latitude coordinates) in radians inline static double centralAngle(const Point2& Alonlat, const Point2& Blonlat, bool normalise_angle = false) { return Sphere::centralAngle(Alonlat, Blonlat, normalise_angle); } /// Great-circle central angle between two points (Cartesian coordinates) in radians inline static double centralAngle(const Point3& A, const Point3& B) { return Sphere::centralAngle(DATUM::radius(), A, B); } /// Great-circle distance between two points (longitude/latitude coordinates) in metres inline static double distance(const Point2& Alonlat, const Point2& Blonlat) { return Sphere::distance(DATUM::radius(), Alonlat, Blonlat); } /// Great-circle distance between two points (Cartesian coordinates) in metres inline static double distance(const Point3& A, const Point3& B) { return Sphere::distance(DATUM::radius(), A, B); } /// Surface area in square metres inline static double area() { return Sphere::area(DATUM::radius()); } /// Surface area between parallels and meridians defined by two points (longitude/latitude coordinates) in square /// metres inline static double area(const Point2& WestNorth, const Point2& EastSouth) { return Sphere::area(DATUM::radius(), WestNorth, EastSouth); } // Great-circle intermediate latitude provided two circle points (A, B) and intermediate longitude (C) in degrees inline static double greatCircleLatitudeGivenLongitude(const Point2& Alonlat, const Point2& Blonlat, const double& Clon) { return Sphere::greatCircleLatitudeGivenLongitude(Alonlat, Blonlat, Clon); } // Great-circle intermediate longitude(s) provided two circle points (A, B) and intermediate latitude (C) in degrees inline static void greatCircleLongitudeGivenLatitude(const Point2& Alonlat, const Point2& Blonlat, const double& Clat, double& Clon1, double& Clon2) { return Sphere::greatCircleLongitudeGivenLatitude(Alonlat, Blonlat, Clat, Clon1, Clon2); } // Convert spherical coordinates to Cartesian inline static void convertSphericalToCartesian(const Point2& Alonlat, Point3& B, double height = 0., bool normalise_angle = false) { Sphere::convertSphericalToCartesian(DATUM::radius(), Alonlat, B, height, normalise_angle); } // Convert Cartesian coordinates to spherical inline static void convertCartesianToSpherical(const Point3& A, Point2& Blonlat) { Sphere::convertCartesianToSpherical(DATUM::radius(), A, Blonlat); } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry #endif eckit-2.0.7/src/eckit/geometry/Sphere.cc0000664000175000017500000001700315161702250020261 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geometry/Sphere.h" #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/geometry/CoordinateHelpers.h" #include "eckit/geometry/GreatCircle.h" #include "eckit/geometry/Point2.h" #include "eckit/geometry/Point3.h" #include "eckit/types/FloatCompare.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- static const double radians_to_degrees = 180. * M_1_PI; static const double degrees_to_radians = M_PI / 180.; inline double squared(double x) { return x * x; } //---------------------------------------------------------------------------------------------------------------------- double Sphere::centralAngle(const Point2& Alonlat, const Point2& Blonlat, bool normalise_angle) { /* * Δσ = atan( ((cos(ϕ2) * sin(Δλ))^2 + (cos(ϕ1) * sin(ϕ2) - sin(ϕ1) * cos(ϕ2) * cos(Δλ))^2) / * (sin(ϕ1) * sin(ϕ2) + cos(ϕ1) * cos(ϕ2) * cos(Δλ)) ) * * @article{doi:10.1179/sre.1975.23.176.88, * author = {T. Vincenty}, * title = {Direct and Inverse Solutions of Geodesics on the Ellipsoid With Application of Nested Equations}, * journal = {Survey Review}, * volume = {23}, * number = {176}, * pages = {88-93}, * year = {1975}, * doi = {10.1179/sre.1975.23.176.88} * } */ if (!normalise_angle) { assert_latitude_range(Alonlat[1]); assert_latitude_range(Blonlat[1]); } const Point2 alonlat = canonicaliseOnSphere(Alonlat); const Point2 blonlat = canonicaliseOnSphere(Blonlat); const double phi1 = degrees_to_radians * alonlat[1]; const double phi2 = degrees_to_radians * blonlat[1]; const double lambda = degrees_to_radians * (blonlat[0] - alonlat[0]); const double cos_phi1 = std::cos(phi1); const double sin_phi1 = std::sin(phi1); const double cos_phi2 = std::cos(phi2); const double sin_phi2 = std::sin(phi2); const double cos_lambda = std::cos(lambda); const double sin_lambda = std::sin(lambda); const double angle = std::atan2( std::sqrt(squared(cos_phi2 * sin_lambda) + squared(cos_phi1 * sin_phi2 - sin_phi1 * cos_phi2 * cos_lambda)), sin_phi1 * sin_phi2 + cos_phi1 * cos_phi2 * cos_lambda); if (types::is_approximately_equal(angle, 0.)) { return 0.; } ASSERT(angle > 0.); return angle; } double Sphere::centralAngle(const double& radius, const Point3& A, const Point3& B) { ASSERT(radius > 0.); // Δσ = 2 * asin( chord / 2 ) const double d2 = Point3::distance2(A, B); if (types::is_approximately_equal(d2, 0.)) { return 0.; } const double chord = std::sqrt(d2) / radius; const double angle = std::asin(chord * 0.5) * 2.; return angle; } double Sphere::distance(const double& radius, const Point2& Alonlat, const Point2& Blonlat) { return radius * centralAngle(Alonlat, Blonlat, true); } double Sphere::distance(const double& radius, const Point3& A, const Point3& B) { return radius * centralAngle(radius, A, B); } double Sphere::area(const double& radius) { ASSERT(radius > 0.); return 4. * M_PI * radius * radius; } double Sphere::area(const double& radius, const Point2& WestNorth, const Point2& EastSouth) { ASSERT(radius > 0.); // Set longitude fraction double W = WestNorth[0]; double E = normalise_angle(EastSouth[0], W); double longitude_range( types::is_approximately_equal(W, E) && !types::is_approximately_equal(EastSouth[0], WestNorth[0]) ? 360. : E - W); ASSERT(longitude_range <= 360.); double longitude_fraction = longitude_range / 360.; // Set latitude fraction double N = WestNorth[1]; double S = EastSouth[1]; ASSERT(-90. <= N && N <= 90.); ASSERT(-90. <= S && S <= 90.); ASSERT(N >= S); double latitude_fraction = 0.5 * (std::sin(degrees_to_radians * N) - std::sin(degrees_to_radians * S)); // Calculate area return area(radius) * latitude_fraction * longitude_fraction; } double Sphere::greatCircleLatitudeGivenLongitude(const Point2& Alonlat, const Point2& Blonlat, const double& Clon) { GreatCircle gc(Alonlat, Blonlat); auto lat = gc.latitude(Clon); return lat.size() == 1 ? lat[0] : std::numeric_limits::signaling_NaN(); } void Sphere::greatCircleLongitudeGivenLatitude(const Point2& Alonlat, const Point2& Blonlat, const double& Clat, double& Clon1, double& Clon2) { GreatCircle gc(Alonlat, Blonlat); auto lon = gc.longitude(Clat); Clon1 = lon.size() > 0 ? lon[0] : std::numeric_limits::signaling_NaN(); Clon2 = lon.size() > 1 ? lon[1] : std::numeric_limits::signaling_NaN(); } void Sphere::convertSphericalToCartesian(const double& radius, const Point2& Alonlat, Point3& B, double height, bool normalise_angle) { ASSERT(radius > 0.); /* * See https://en.wikipedia.org/wiki/Reference_ellipsoid#Coordinates * numerical conditioning for both ϕ (poles) and λ (Greenwich/Date Line). * * cos α = sqrt( 1 - sin^2 α) is better conditioned than explicit cos α, and * coupled with λ in [-180°, 180°[ the accuracy of the trigonometric * functions is the same (before converting/multiplying its angle argument * to radian) and explicitly chosing -180° over 180° for longitude. * * These three conditionings combined project very accurately to the sphere * poles and quadrants. */ if (!normalise_angle) { assert_latitude_range(Alonlat[1]); } const Point2 alonlat = canonicaliseOnSphere(Alonlat, -180.); const double lambda_deg = alonlat[0]; const double lambda = degrees_to_radians * lambda_deg; const double phi = degrees_to_radians * alonlat[1]; const double sin_phi = std::sin(phi); const double cos_phi = std::sqrt(1. - sin_phi * sin_phi); const double sin_lambda = std::abs(lambda_deg) < 180. ? std::sin(lambda) : 0.; const double cos_lambda = std::abs(lambda_deg) > 90. ? std::cos(lambda) : std::sqrt(1. - sin_lambda * sin_lambda); B[0] = (radius + height) * cos_phi * cos_lambda; B[1] = (radius + height) * cos_phi * sin_lambda; B[2] = (radius + height) * sin_phi; } void Sphere::convertCartesianToSpherical(const double& radius, const Point3& A, Point2& Blonlat) { ASSERT(radius > 0.); // numerical conditioning for both z (poles) and y const double x = A[0]; const double y = types::is_approximately_equal(A[1], 0.) ? 0. : A[1]; const double z = std::min(radius, std::max(-radius, A[2])) / radius; Blonlat[0] = radians_to_degrees * std::atan2(y, x); Blonlat[1] = radians_to_degrees * std::asin(z); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry eckit-2.0.7/src/eckit/geometry/GreatCircle.cc0000664000175000017500000001107715161702250021224 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geometry/GreatCircle.h" #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/types/FloatCompare.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- static double normalise_longitude(double a, const double& minimum) { while (a < minimum) { a += 360; } while (a >= minimum + 360) { a -= 360; } return a; } static const double radians_to_degrees = 180. * M_1_PI; static const double degrees_to_radians = M_PI / 180.; static constexpr std::streamsize max_digits10 = std::numeric_limits::max_digits10; static bool pole(const double lat) { return types::is_approximately_equal(std::abs(lat), 90.); } //---------------------------------------------------------------------------------------------------------------------- GreatCircle::GreatCircle(const Point2& Alonlat, const Point2& Blonlat) : A_(Alonlat), B_(Blonlat) { using namespace std; using types::is_approximately_equal; const bool Apole = pole(A_[1]); const bool Bpole = pole(B_[1]); const double lon12_deg = normalise_longitude(A_[0] - B_[0], -180); const bool lon_same = Apole || Bpole || is_approximately_equal(lon12_deg, 0.); const bool lon_opposite = Apole || Bpole || is_approximately_equal(abs(lon12_deg), 180.); const bool lat_same = is_approximately_equal(A_[1], B_[1]); const bool lat_opposite = is_approximately_equal(A_[1], -B_[1]); if ((lat_same && lon_same) || (lat_opposite && lon_opposite)) { ostringstream oss; oss.precision(max_digits10); oss << "Great circle cannot be defined by points collinear with the centre, " << A_ << " and " << B_; throw BadValue(oss.str(), Here()); } crossesPoles_ = lon_same || lon_opposite; } std::vector GreatCircle::latitude(double lon) const { using namespace std; if (crossesPoles()) { return {}; } const double lat1 = degrees_to_radians * A_[1]; const double lat2 = degrees_to_radians * B_[1]; const double lambda1p = degrees_to_radians * (lon - A_[0]); const double lambda2p = degrees_to_radians * (lon - B_[0]); const double lambda = degrees_to_radians * normalise_longitude(B_[0] - A_[0], -180); double lat = atan((tan(lat2) * sin(lambda1p) - tan(lat1) * sin(lambda2p)) / (sin(lambda))); return {radians_to_degrees * lat}; } std::vector GreatCircle::longitude(double lat) const { using namespace std; using types::is_approximately_equal; if (crossesPoles()) { const double lon = pole(A_[1]) ? B_[0] : A_[0]; return pole(lat) ? vector{lon} : vector{lon, lon + 180}; } const double lon12 = degrees_to_radians * normalise_longitude(A_[0] - B_[0], -180); const double lon1 = degrees_to_radians * A_[0]; const double lat1 = degrees_to_radians * A_[1]; const double lat2 = degrees_to_radians * B_[1]; const double lat3 = degrees_to_radians * lat; const double X = sin(lat1) * cos(lat2) * sin(lon12); const double Y = sin(lat1) * cos(lat2) * cos(lon12) - cos(lat1) * sin(lat2); if (is_approximately_equal(X, 0.) && is_approximately_equal(Y, 0.)) { return {}; // parallel (that is, equator) } const double lon0 = lon1 + atan2(Y, X); const double C = cos(lat1) * cos(lat2) * tan(lat3) * sin(lon12) / sqrt(X * X + Y * Y); if (is_approximately_equal(C, -1.)) { return {radians_to_degrees * (lon0 + M_PI)}; } if (is_approximately_equal(C, 1.)) { return {radians_to_degrees * lon0}; } if (-1 < C && C < 1) { const double dlon = acos(C); return {radians_to_degrees * (lon0 - dlon + 2 * M_PI), radians_to_degrees * (lon0 + dlon)}; } return {}; } bool GreatCircle::crossesPoles() const { return crossesPoles_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry eckit-2.0.7/src/eckit/geometry/CoordinateHelpers.cc0000664000175000017500000000431115161702250022443 0ustar alastairalastair/* * (C) Copyright 2023 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/geometry/CoordinateHelpers.h" #include "eckit/geometry/Point2.h" namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- inline double modulo(const double a, const double b) { return a - b * std::floor(a / b); } double normalise_angle(double a, const double minimum) { constexpr double treshold = 4. * 360.; const double diff = a - minimum; if (std::abs(diff) > treshold) { // This formula is not bit-dentical with original below, // but is faster for very large values of 'a'. // The treshold tries to capture values of 'a' which need to // stay bit-identical. return minimum + modulo(diff, 360.); } // The original function while (a < minimum) { a += 360.; } while (a >= minimum + 360.) { a -= 360.; } return a; } //---------------------------------------------------------------------------------------------------------------------- Point2 canonicaliseOnSphere(const Point2& lonlat, const double minimum_lon) { const double lat = normalise_angle(lonlat[1], -90.); const bool across_pole = (lat > 90.); if (!across_pole) { return {normalise_angle(lonlat[0], minimum_lon), lat}; } return {normalise_angle(lonlat[0] + 180., minimum_lon), 180. - lat}; } //---------------------------------------------------------------------------------------------------------------------- void assert_latitude_range(double lat) { if (!(-90. <= lat && lat <= 90.)) { std::ostringstream oss; oss.precision(std::numeric_limits::max_digits10); oss << "Invalid latitude " << lat; throw BadValue(oss.str(), Here()); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry eckit-2.0.7/src/eckit/geometry/CoordinateHelpers.h0000664000175000017500000000353715161702250022316 0ustar alastairalastair/* * (C) Copyright 2023 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ #ifndef CoordinateHelpers_H #define CoordinateHelpers_H namespace eckit::geometry { class Point2; //---------------------------------------------------------------------------------------------------------------------- /// Shift angle in increments of 360° until it lies in [minimum, minimum+360°). /// /// Inputs angle and minimum are in degrees, returned angle is in degrees. double normalise_angle(double angle, double minimum); //---------------------------------------------------------------------------------------------------------------------- /// Shift input point on sphere so its longitude lies in [minimum_lon, minimum_lon+360°) /// and its latitude lies in [-90°, 90°]. /// /// Latitudes outside the canonical interval [-90°,90°] are first shifted into the interval /// [-90°,270°], then any points with latitudes in [90°,270°] are flagged as "across the pole". /// Such points are re-labeled with equivalent coordinates that lie within the canonical coordinate /// patch by the transformation: (λ, ϕ) -> (λ+180°, 180°-ϕ). /// /// Finally, the longitude is shifted into [minimum_lon, minimum_lon+360°). /// /// Inputs lonlat and minimum_lon are in degrees, returned angles are in degrees. Point2 canonicaliseOnSphere(const Point2& lonlat, double minimum_lon = 0.); //---------------------------------------------------------------------------------------------------------------------- /// Assert latitude lies in [-90°, 90°]. void assert_latitude_range(double lat); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry #endif // CoordinateHelpers_H eckit-2.0.7/src/eckit/geometry/Point2.h0000664000175000017500000000313015161702250020044 0ustar alastairalastair#ifndef eckit_geometry_Point2_h #define eckit_geometry_Point2_h #include #include "eckit/geometry/KPoint.h" //------------------------------------------------------------------------------------------------------ namespace eckit { class Value; } //------------------------------------------------------------------------------------------------------ namespace eckit::geometry { //------------------------------------------------------------------------------------------------------ class Point2 : public eckit::geometry::KPoint<2> { using BasePoint = KPoint<2>; public: Point2() : BasePoint() {} Point2(const BasePoint& p) : BasePoint(p) {} Point2(const double* p) : BasePoint(p) {} Point2(double x, double y) : BasePoint() { x_[XX] = x; x_[YY] = y; } double x() const { return x_[0]; } double y() const { return x_[1]; } double x(size_t axis) const { return eckit::geometry::KPoint<2>::x(axis); } double operator[](const size_t& i) const { assert(i < 2); return x_[i]; } double& operator[](const size_t& i) { assert(i < 2); return x_[i]; } template void assign(const T& p) { x_[XX] = p[XX]; x_[YY] = p[YY]; } operator eckit::Value() const; }; //------------------------------------------------------------------------------------------------------ bool points_equal(const Point2&, const Point2&); //------------------------------------------------------------------------------------------------------ } // namespace eckit::geometry #endif eckit-2.0.7/src/eckit/geometry/Point3.cc0000664000175000017500000000120615161702250020205 0ustar alastairalastair#include "eckit/geometry/Point3.h" #include "eckit/types/FloatCompare.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- bool points_equal(const Point3& a, const Point3& b) { return eckit::types::is_approximately_equal(Point3::distance2(a, b), 0.0); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry eckit-2.0.7/src/eckit/geometry/Point3.h0000664000175000017500000000300415161702250020045 0ustar alastairalastair#ifndef eckit_geometry_Point3_h #define eckit_geometry_Point3_h #include "eckit/geometry/KPoint.h" //------------------------------------------------------------------------------------------------------ namespace eckit::geometry { //------------------------------------------------------------------------------------------------------ class Point3 : public eckit::geometry::KPoint<3> { using BasePoint = KPoint<3>; public: Point3() : BasePoint() {} Point3(const BasePoint& p) : BasePoint(p) {} Point3(const double* p) : BasePoint(p) {} Point3(double x, double y, double z) : BasePoint() { x_[XX] = x; x_[YY] = y; x_[ZZ] = z; } double operator[](const size_t& i) const { assert(i < 3); return x_[i]; } double& operator[](const size_t& i) { assert(i < 3); return x_[i]; } template void assign(const T& p) { x_[XX] = p[XX]; x_[YY] = p[YY]; x_[ZZ] = p[ZZ]; } static Point3 cross(const Point3& p1, const Point3& p2) { return Point3(p1[YY] * p2[ZZ] - p1[ZZ] * p2[YY], p1[ZZ] * p2[XX] - p1[XX] * p2[ZZ], p1[XX] * p2[YY] - p1[YY] * p2[XX]); } }; //------------------------------------------------------------------------------------------------------ bool points_equal(const Point3&, const Point3&); //------------------------------------------------------------------------------------------------------ } // namespace eckit::geometry #endif eckit-2.0.7/src/eckit/geometry/GreatCircle.h0000664000175000017500000000262015161702250021060 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef GreatCircle_H #define GreatCircle_H #include #include "eckit/geometry/Point2.h" //------------------------------------------------------------------------------------------------------ namespace eckit::geometry { //------------------------------------------------------------------------------------------------------ class GreatCircle { public: /// Great circle given two points in geographic coordinates GreatCircle(const Point2&, const Point2&); /// Great circle latitude given longitude, see http://www.edwilliams.org/avform.htm#Int std::vector latitude(double lon) const; /// Great circle longitude given latitude, see http://www.edwilliams.org/avform.htm#Par std::vector longitude(double lat) const; bool crossesPoles() const; private: const Point2 A_; const Point2 B_; bool crossesPoles_; }; //------------------------------------------------------------------------------------------------------ } // namespace eckit::geometry #endif eckit-2.0.7/src/eckit/geometry/EllipsoidOfRevolution.h0000664000175000017500000000250315161702250023174 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef EllipsoidOfRevolution_H #define EllipsoidOfRevolution_H //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- class Point2; class Point3; //---------------------------------------------------------------------------------------------------------------------- struct EllipsoidOfRevolution { // Convert elliptic coordinates to Cartesian static void convertSphericalToCartesian(const double& radiusA, const double& radiusB, const Point2& Alonlat, Point3& B, double height = 0., bool normalise_angle = false); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry #endif eckit-2.0.7/src/eckit/geometry/Point2.cc0000664000175000017500000000151615161702250020210 0ustar alastairalastair#include #include "eckit/geometry/Point2.h" #include "eckit/types/FloatCompare.h" #include "eckit/value/Value.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- bool points_equal(const Point2& a, const Point2& b) { return eckit::types::is_approximately_equal(Point2::distance2(a, b), 0.0); } Point2::operator Value() const { std::vector pts; pts.push_back(x_[XX]); pts.push_back(x_[YY]); return Value::makeList(pts); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry eckit-2.0.7/src/eckit/geometry/KPoint.cc0000664000175000017500000000142115161702250020234 0ustar alastairalastair #include #include "eckit/geometry/KPoint.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit::geometry { //---------------------------------------------------------------------------------------------------------------------- template void KPoint::print(std::ostream& s) const { char z = '{'; for (size_t i = 0; i < SIZE; ++i) { s << z << x_[i]; z = ','; } s << '}'; } template void KPoint<2>::print(std::ostream&) const; template void KPoint<3>::print(std::ostream&) const; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::geometry eckit-2.0.7/src/eckit/geometry/CMakeLists.txt0000664000175000017500000000107415161702250021265 0ustar alastairalastairlist( APPEND eckit_geometry_srcs CoordinateHelpers.cc CoordinateHelpers.h EllipsoidOfRevolution.cc EllipsoidOfRevolution.h GreatCircle.cc GreatCircle.h KPoint.cc KPoint.h Point2.cc Point2.h Point3.cc Point3.h Sphere.cc Sphere.h SphereT.h UnitSphere.h polygon/LonLatPolygon.cc polygon/LonLatPolygon.h polygon/Polygon.cc polygon/Polygon.h ) ecbuild_add_library( TARGET eckit_geometry TYPE SHARED INSTALL_HEADERS ALL HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/geometry SOURCES ${eckit_geometry_srcs} PUBLIC_LIBS eckit ) eckit-2.0.7/src/eckit/geometry/UnitSphere.h0000664000175000017500000000230615161702250020763 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef UnitSphere_H #define UnitSphere_H #include "eckit/geometry/SphereT.h" //------------------------------------------------------------------------------------------------------ namespace eckit::geometry { //------------------------------------------------------------------------------------------------------ /// Definition of a unit datum struct DatumUnit { /* C++-11: static constexpr double radius() { return 1.; } */ static double radius() { return 1.; } }; //------------------------------------------------------------------------------------------------------ /// Definition of a unit sphere using UnitSphere = SphereT; //------------------------------------------------------------------------------------------------------ } // namespace eckit::geometry #endif eckit-2.0.7/src/eckit/geometry/KPoint.h0000664000175000017500000001727415161702250020113 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef KPoint_H #define KPoint_H #include #include #include #include #include #include //------------------------------------------------------------------------------------------------------ namespace eckit::geometry { //------------------------------------------------------------------------------------------------------ enum XYZCOORDS { XX = 0, YY = 1, ZZ = 2 }; enum LLCOORDS { LON = XX, LAT = YY }; /// A generic point in K dimension cartesian space template class KPoint { protected: double x_[SIZE] = {0}; public: static const size_t DIMS = SIZE; double x(size_t axis) const { return x_[axis]; } KPoint() {} KPoint(const double* x) { std::copy(x, x + dimensions(), x_); } template explicit KPoint(Container c) { std::copy(c.begin(), c.end(), x_); } const KPoint& point() const { return *this; } KPoint& point() { return *this; } double* data() { return x_; } const double* data() const { return x_; } double operator()(const size_t& i) const { assert(i < SIZE); return x_[i]; } bool operator<(const KPoint& other) const { return std::lexicographical_compare(x_, x_ + SIZE, other.x_, other.x_ + SIZE); } static size_t dimensions() { return SIZE; } void print(std::ostream& s) const; friend std::ostream& operator<<(std::ostream& s, const KPoint& p) { p.print(s); return s; } static double distance(const KPoint& p1, const KPoint& p2) { double d = 0; for (size_t i = 0; i < dimensions(); i++) { double dx = p1.x_[i] - p2.x_[i]; d += dx * dx; } return std::sqrt(d); } double distance(const KPoint& p) const { return distance(*this, p); } static double distance2(const KPoint& p1, const KPoint& p2) { double d = 0; for (size_t i = 0; i < dimensions(); i++) { double dx = p1.x_[i] - p2.x_[i]; d += dx * dx; } return d; } double distance2(const KPoint& p) const { return distance2(*this, p); } static bool equal(const KPoint& p1, const KPoint& p2) { for (size_t i = 0; i < dimensions(); i++) { if (p1.x_[i] != p2.x_[i]) { return false; } } return true; } bool operator==(const KPoint& other) const { return equal(*this, other); } bool operator!=(const KPoint& other) const { return !equal(*this, other); } static double norm(const KPoint& p1) { double n = 0.0; for (size_t i = 0; i < dimensions(); i++) { double dx = p1.x_[i]; n += dx * dx; } return std::sqrt(n); } // Distance along one axis static double distance(const KPoint& p1, const KPoint& p2, unsigned int axis) { return std::abs(p1.x_[axis] - p2.x_[axis]); } // For projecting a point on a line static double dot(const KPoint& p1, const KPoint& p2) { double m = 0.0; for (size_t i = 0; i < dimensions(); i++) { m += p1.x_[i] * p2.x_[i]; } return m; } static KPoint add(const KPoint& p1, const KPoint& p2) { KPoint q(p1); for (size_t i = 0; i < dimensions(); i++) { q.x_[i] += p2.x_[i]; } return q; } static KPoint middle(const KPoint& p1, const KPoint& p2) { KPoint q(p1); for (size_t i = 0; i < dimensions(); i++) { q.x_[i] += p2.x_[i]; q.x_[i] /= 2.0; } return q; } static KPoint sub(const KPoint& p1, const KPoint& p2) { KPoint q(p1); for (size_t i = 0; i < dimensions(); i++) { q.x_[i] -= p2.x_[i]; } return q; } static KPoint mul(const KPoint& p, double m) { KPoint q(p); for (size_t i = 0; i < dimensions(); i++) { q.x_[i] *= m; } return q; } static KPoint div(const KPoint& p, double m) { KPoint q(p); for (size_t i = 0; i < dimensions(); i++) { q.x_[i] /= m; } return q; } static KPoint componentsMin(const KPoint& p1, const KPoint& p2) { KPoint q; for (size_t i = 0; i < dimensions(); i++) { q.x_[i] = std::min(p1.x_[i], p2.x_[i]); } return q; } static KPoint componentsMax(const KPoint& p1, const KPoint& p2) { KPoint q; for (size_t i = 0; i < dimensions(); i++) { q.x_[i] = std::max(p1.x_[i], p2.x_[i]); } return q; } static KPoint normalize(const KPoint& p) { KPoint zero; return div(p, distance(p, zero)); } template static typename Container::value_type mean(const Container& points) { typename Container::const_iterator j = points.begin(); typename Container::value_type result(*j); ++j; for (; j != points.end(); ++j) { for (size_t i = 0; i < dimensions(); i++) { result.point().x_[i] += (*j).point().x_[i]; } } for (size_t i = 0; i < dimensions(); i++) { result.point().x_[i] /= points.size(); } return result; } static KPoint symetrical(const KPoint& w, const KPoint& c) { KPoint result(w); for (size_t i = 0; i < dimensions(); i++) { result.x_[i] -= (c.x_[i] - w.x_[i]); } return result; } const double* begin() const { return x_; } const double* end() const { return x_ + dimensions(); } KPoint operator+(const KPoint& other) const { return add(*this, other); } KPoint operator-(const KPoint& other) const { return sub(*this, other); } KPoint operator*(const double s) const { return mul(*this, s); } void normalize(const KPoint& offset, const KPoint& scale) { for (size_t i = 0; i < DIMS; ++i) { x_[i] = (x_[i] - offset.x_[i]) / scale.x_[i]; } } template static void normalizeAll(Container& c, KPoint& offset, KPoint& scale) { std::vector mins(DIMS, std::numeric_limits::max()); std::vector maxs(DIMS, -std::numeric_limits::max()); for (typename Container::const_iterator j = c.begin(); j != c.end(); ++j) { const typename Container::value_type& v = (*j); for (size_t i = 0; i < DIMS; ++i) { mins[i] = std::min(mins[i], v.point().x_[i]); maxs[i] = std::max(maxs[i], v.point().x_[i]); } } for (size_t i = 0; i < DIMS; ++i) { maxs[i] -= mins[i]; } for (typename Container::iterator j = c.begin(); j != c.end(); ++j) { typename Container::value_type& v = (*j); for (size_t i = 0; i < DIMS; ++i) { v.point().x_[i] = (v.point().x_[i] - mins[i]) / maxs[i]; } } offset = KPoint(mins); scale = KPoint(maxs); } }; //------------------------------------------------------------------------------------------------------ template const size_t KPoint::DIMS; //------------------------------------------------------------------------------------------------------ } // namespace eckit::geometry #endif eckit-2.0.7/src/eckit/geo/0000775000175000017500000000000015161702250015442 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/range/0000775000175000017500000000000015161702250016536 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/range/GaussianLatitude.cc0000664000175000017500000000465015161702250022320 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/range/GaussianLatitude.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/util.h" #include "eckit/geo/util/mutex.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::range { static util::recursive_mutex MUTEX; GaussianLatitude::GaussianLatitude(size_t N, bool increasing) : GaussianLatitude(N, std::vector(util::gaussian_latitudes(N, increasing))) {} bool GaussianLatitude::includesNorthPole() const { const auto increasing = a() < b(); const auto& lats(util::gaussian_latitudes(N_, increasing)); return increasing ? types::is_approximately_equal(b(), lats.back()) : types::is_approximately_equal(a(), lats.front()); } bool GaussianLatitude::includesSouthPole() const { const auto increasing = a() < b(); const auto& lats(util::gaussian_latitudes(N_, increasing)); return !increasing ? types::is_approximately_equal(b(), lats.back()) : types::is_approximately_equal(a(), lats.front()); } GaussianLatitude* GaussianLatitude::make_cropped_range(double crop_a, double crop_b) const { ASSERT((a() < b() && crop_a <= crop_b) || (a() > b() && crop_a >= crop_b) || (types::is_approximately_equal(a(), b()) && types::is_approximately_equal(crop_a, crop_b))); auto v = values(); if ((a() < b()) && (a() < crop_a || crop_b < b())) { auto [from, to] = util::monotonic_crop(v, crop_a, crop_b); v.erase(v.begin() + to, v.end()); v.erase(v.begin(), v.begin() + from); } else if ((b() < a()) && (b() < crop_b || crop_a < a())) { auto [from, to] = util::monotonic_crop(v, crop_b, crop_a); v.erase(v.begin() + to, v.end()); v.erase(v.begin(), v.begin() + from); } return new GaussianLatitude(N_, std::move(v)); } const std::vector& GaussianLatitude::values() const { util::lock_guard lock(MUTEX); return values_.empty() ? util::gaussian_latitudes(N_, a() < b()) : values_; } } // namespace eckit::geo::range eckit-2.0.7/src/eckit/geo/range/GaussianLatitude.h0000664000175000017500000000254315161702250022161 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Range.h" namespace eckit::geo::range { class GaussianLatitude final : public Range { public: // -- Constructors explicit GaussianLatitude(size_t N, bool increasing); // -- Methods size_t N() const { return N_; } // -- Overridden methods [[nodiscard]] GaussianLatitude* make_cropped_range(double crop_a, double crop_b) const override; [[nodiscard]] const std::vector& values() const override; size_t size() const override { return values_.size(); } double a() const override { return values_.front(); } double b() const override { return values_.back(); } bool includesNorthPole() const override; bool includesSouthPole() const override; private: // -- Constructors GaussianLatitude(size_t N, std::vector&& values) : N_(N), values_(values) {} // -- Members const size_t N_; const std::vector values_; }; } // namespace eckit::geo::range eckit-2.0.7/src/eckit/geo/range/Regular.h0000664000175000017500000000752415161702250020320 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/Range.h" #include "eckit/types/Fraction.h" namespace eckit::geo::range { class Regular : public Range { public: // -- Types struct Implementation { Implementation(Fraction inc, Fraction a, Fraction b, Fraction ref); Implementation(const Implementation&) = default; Implementation(Implementation&&) = default; virtual ~Implementation() = default; Implementation& operator=(const Implementation&) = default; Implementation& operator=(Implementation&&) = default; virtual Regular* make_cropped_range(double, double) const; virtual bool periodic() const; virtual bool includesNorthPole() const; virtual bool includesSouthPole() const; Fraction min() const; Fraction max() const; Fraction inc_; Fraction shift_; Fraction a_; Fraction b_; size_t size_ = 0; }; // -- Overridden methods [[nodiscard]] const std::vector& values() const override; [[nodiscard]] Regular* make_cropped_range(double crop_a, double crop_b) const override { return impl_->make_cropped_range(crop_a, crop_b); } size_t size() const override { return impl_->size_; } double a() const override { return impl_->a_; } double b() const override { return impl_->b_; } Fraction increment() const override { return impl_->inc_; } bool periodic() const override { return impl_->periodic(); } bool includesNorthPole() const override { return impl_->includesNorthPole(); } bool includesSouthPole() const override { return impl_->includesSouthPole(); } // -- Methods Fraction shift() const { return impl_->shift_; } Fraction af() const { return impl_->a_; } Fraction bf() const { return impl_->b_; } protected: // -- Constructors explicit Regular(Implementation*); private: // -- Members std::shared_ptr impl_; }; struct RegularLatitude : Regular { explicit RegularLatitude(Fraction inc, Fraction a, Fraction b, Fraction ref = {}); explicit RegularLatitude(double inc, double a, double b, double ref = 0) : RegularLatitude(Fraction{inc}, Fraction{a}, Fraction{b}, Fraction{ref}) {} [[nodiscard]] RegularLatitude* make_cropped_range(double crop_a, double crop_b) const override; }; struct RegularLongitude : Regular { explicit RegularLongitude(Fraction inc, Fraction a, Fraction b, Fraction ref = {}); explicit RegularLongitude(double inc, double a, double b, double ref = 0) : RegularLongitude(Fraction{inc}, Fraction{a}, Fraction{b}, Fraction{ref}) {} [[nodiscard]] static RegularLongitude* make_empty_range(Fraction a, Fraction b); [[nodiscard]] static RegularLongitude* make_empty_range(double a, double b) { return make_empty_range(Fraction{a}, Fraction{b}); } [[nodiscard]] RegularLongitude* make_cropped_range(double crop_a, double crop_b) const override; private: explicit RegularLongitude(Implementation* impl) : Regular(impl) {} }; struct RegularXY : Regular { explicit RegularXY(Fraction inc, Fraction a, Fraction b, Fraction ref = {}); explicit RegularXY(double inc, double a, double b, double ref = 0) : RegularXY(Fraction{inc}, Fraction{a}, Fraction{b}, Fraction{ref}) {} [[nodiscard]] RegularXY* make_cropped_range(double crop_a, double crop_b) const override; }; } // namespace eckit::geo::range eckit-2.0.7/src/eckit/geo/range/Regular.cc0000664000175000017500000001453015161702250020451 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/range/Regular.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/PointLonLat.h" #include "eckit/geo/util.h" #include "eckit/geo/util/mutex.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::range { namespace detail { Fraction adjust(const Fraction& target, const Fraction& inc, bool up) { auto r = target / inc; auto n = r.integralPart() + ((r.integer() || (r > 0) != up) ? 0 : up ? 1 : -1); return n * inc; }; size_t N(Fraction inc, Fraction a, Fraction b) { return static_cast(inc == 0 ? 1 : (Fraction::abs((b - a) / inc)).integralPart() + 1); } struct RegularXY final : Regular::Implementation { using Implementation::Implementation; }; struct RegularLatitude final : Regular::Implementation { using Implementation::Implementation; Regular* make_cropped_range(double crop_a, double crop_b) const override { std::unique_ptr x{Implementation::make_cropped_range(crop_a, crop_b)}; return new range::RegularLatitude(x->increment(), x->af(), x->bf(), x->af()); } bool includesNorthPole() const override { return types::is_approximately_greater_or_equal(max() + Fraction::abs(inc_), NORTH_POLE.lat()); } bool includesSouthPole() const override { return types::is_approximately_lesser_or_equal(min() - Fraction::abs(inc_), SOUTH_POLE.lat()); } }; struct RegularLongitude final : Regular::Implementation { RegularLongitude(Fraction inc, Fraction a, Fraction b, Fraction ref) : Implementation(inc, a, b, ref) { if (size_ == 1) { return; } if (Fraction::abs(b_ - a_ + inc_) >= PERIOD) { b_ = a_ + adjust(PERIOD, inc_, a_ < b_) - inc_; size_ = N(inc_, a_, b_); periodic_ = true; } } Regular* make_cropped_range(double crop_a, double crop_b) const override { if (periodic()) { return new range::RegularLongitude(inc_, crop_a, crop_b, a_); } std::unique_ptr x{Implementation::make_cropped_range(crop_a, crop_b)}; return size_ == 0 ? range::RegularLongitude::make_empty_range(x->af(), x->bf()) : new range::RegularLongitude(x->increment(), x->af(), x->bf(), x->af()); } static RegularLongitude* make_empty(Fraction a, Fraction b) { auto* impl(new RegularLongitude(b - a, a, b, a)); ASSERT(impl != nullptr); impl->size_ = 0; return impl; } bool periodic() const override { return periodic_; } bool periodic_ = false; const static Fraction PERIOD; }; const Fraction RegularLongitude::PERIOD{360, 1}; } // namespace detail Regular::Implementation::Implementation(Fraction inc, Fraction a, Fraction b, Fraction ref) : inc_(inc), a_(a), b_(b) { if (types::is_approximately_equal(inc_, 0.) || types::is_approximately_lesser_or_equal((b_ - a_) * inc_, 0.)) { size_ = 1; inc_ = {}; b_ = a_; return; } shift_ = (ref / inc).decimalPart() * inc; a_ = shift_ + detail::adjust(a_ - shift_, inc_, true); b_ = shift_ + detail::adjust(b_ - shift_, inc_, false); ASSERT(a < b == a_ < b_); size_ = detail::N(inc_, a_, b_); ASSERT(size_ > 0); } Regular* Regular::Implementation::make_cropped_range(double crop_a, double crop_b) const { const bool up = a_ <= b_; ASSERT(up ? crop_a <= crop_b : crop_a > crop_b); detail::RegularXY other(inc_, Fraction{crop_a}, Fraction{crop_b}, a_); auto _min = std::max(min(), other.min()); auto _max = std::min(max(), other.max()); return new range::RegularXY{inc_, up ? _min : _max, up ? _max : _min, a_}; } bool Regular::Implementation::periodic() const { NOTIMP; } bool Regular::Implementation::includesNorthPole() const { NOTIMP; } bool Regular::Implementation::includesSouthPole() const { NOTIMP; } Fraction Regular::Implementation::min() const { return std::min(a_, b_); } Fraction Regular::Implementation::max() const { return std::max(a_, b_); } const std::vector& Regular::values() const { static util::recursive_mutex MUTEX; util::lock_guard lock(MUTEX); return util::linspace(a(), b(), size()); } Regular::Regular(Implementation* impl) : impl_(impl) { ASSERT(impl_); } RegularLatitude::RegularLatitude(Fraction inc, Fraction _a, Fraction _b, Fraction ref) : Regular(new detail::RegularLatitude(inc, _a, _b, ref)) { ASSERT(types::is_approximately_equal(a(), b()) ? size() == 1 : size() > 1); ASSERT(SOUTH_POLE.lat() <= min() && min() <= max() && max() <= NORTH_POLE.lat()); } RegularLatitude* RegularLatitude::make_cropped_range(double crop_a, double crop_b) const { auto* ptr = dynamic_cast(Regular::make_cropped_range(crop_a, crop_b)); ASSERT(ptr != nullptr); return ptr; } RegularLongitude::RegularLongitude(Fraction inc, Fraction _a, Fraction _b, Fraction ref) : Regular(new detail::RegularLongitude(inc, _a, _b, ref)) { ASSERT(types::is_approximately_equal(a(), b()) ? size() == 1 : size() > 1); } RegularLongitude* RegularLongitude::make_empty_range(Fraction a, Fraction b) { return new RegularLongitude(detail::RegularLongitude::make_empty(a, b)); } RegularLongitude* RegularLongitude::make_cropped_range(double crop_a, double crop_b) const { auto* ptr = dynamic_cast(Regular::make_cropped_range(crop_a, crop_b)); ASSERT(ptr != nullptr); return ptr; } RegularXY::RegularXY(Fraction inc, Fraction _a, Fraction _b, Fraction ref) : Regular(new detail::RegularXY(inc, _a, _b, ref)) { ASSERT(types::is_approximately_equal(a(), b()) ? size() == 1 : size() > 1); } RegularXY* RegularXY::make_cropped_range(double crop_a, double crop_b) const { auto* ptr = dynamic_cast(Regular::make_cropped_range(crop_a, crop_b)); ASSERT(ptr != nullptr); return ptr; } } // namespace eckit::geo::range eckit-2.0.7/src/eckit/geo/LibEcKitGeo.cc0000664000175000017500000000676015161702250020043 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/LibEcKitGeo.h" #include #include "eckit/config/Resource.h" #include "eckit/eckit_version.h" #include "eckit/filesystem/PathName.h" #include "eckit/geo/eckit_geo_config.h" #include "eckit/utils/StringTools.h" namespace eckit { REGISTER_LIBRARY(LibEcKitGeo); LibEcKitGeo::LibEcKitGeo() : Library("eckit_geo") {} LibEcKitGeo& LibEcKitGeo::instance() { static LibEcKitGeo lib; return lib; } std::vector LibEcKitGeo::shareArea() { static const auto paths = [](const std::string& s) -> std::vector { const auto ss = StringTools::split(":", s); return {ss.begin(), ss.end()}; }(LibResource("eckit-geo-share-area;$ECKIT_GEO_SHARE_AREA", eckit_GEO_SHARE_AREA)); return paths; } std::vector LibEcKitGeo::shareGrid() { static const auto paths = [](const std::string& s) -> std::vector { const auto ss = StringTools::split(":", s); return {ss.begin(), ss.end()}; }(LibResource("eckit-geo-share-grid;$ECKIT_GEO_SHARE_GRID", eckit_GEO_SHARE_GRID)); return paths; } std::vector LibEcKitGeo::shareProjection() { static const auto paths = [](const std::string& s) -> std::vector { const auto ss = StringTools::split(":", s); return {ss.begin(), ss.end()}; }(LibResource("eckit-geo-share-projection;$ECKIT_GEO_SHARE_PROJECTION", eckit_GEO_SHARE_PROJECTION)); return paths; } bool LibEcKitGeo::caching() { static const bool yes{ LibResource("eckit-geo-caching;$ECKIT_GEO_CACHING", eckit_HAVE_GEO_CACHING)}; return yes; } std::string LibEcKitGeo::cacheDir() { static std::string path = PathName{ LibResource("eckit-geo-cache-path;$ECKIT_GEO_CACHE_PATH", eckit_GEO_CACHE_PATH), true}; return path; } std::string LibEcKitGeo::url(const std::string& url_path) { static const std::regex has_scheme(R"(^[a-zA-Z][a-zA-Z0-9+\-.]*://)"); static const std::string url_prefix(LibResource( "eckit-geo-share-url-prefix;$ECKIT_GEO_SHARE_URL_PREFIX", eckit_GEO_SHARE_URL_PREFIX)); auto sep = !url_prefix.empty() && url_prefix.back() != '/' && !url_path.empty() && url_path.front() != '/' ? "/" : ""; return std::regex_search(url_path, has_scheme) ? url_path : url_prefix + sep + url_path; } bool LibEcKitGeo::proj() { static const bool yes{ LibResource("eckit-geo-projection-proj;$ECKIT_GEO_PROJECTION_PROJ", (eckit_HAVE_PROJ != 0) && (eckit_HAVE_GEO_PROJECTION_PROJ_DEFAULT != 0))}; return yes; } const void* LibEcKitGeo::addr() const { return this; } std::string LibEcKitGeo::version() const { return eckit_version_str(); } std::string LibEcKitGeo::gitsha1(unsigned int count) const { std::string sha1(eckit_git_sha1()); return sha1.empty() ? "not available" : sha1.substr(0, std::min(count, 40U)); } } // namespace eckit eckit-2.0.7/src/eckit/geo/Arrangement.h0000664000175000017500000000116315161702250020057 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once namespace eckit::geo { enum Arrangement { NONE = 0, FESOM_C, FESOM_N, ICON_C, ICON_V, ICON_E, ORCA_F, ORCA_T, ORCA_U, ORCA_V, ORCA_W, }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Range.h0000664000175000017500000000255015161702250016651 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/types/Fraction.h" namespace eckit::geo { class Range { public: // -- Constructors Range() = default; Range(const Range&) = default; Range(Range&&) = default; // -- Destructors virtual ~Range() = default; // -- Operators Range& operator=(const Range&) = default; Range& operator=(Range&&) = default; // -- Methods double min() const; double max() const; [[nodiscard]] virtual Range* make_cropped_range(double crop_a, double crop_b) const = 0; [[nodiscard]] virtual const std::vector& values() const = 0; virtual size_t size() const = 0; virtual double a() const = 0; virtual double b() const = 0; virtual Fraction increment() const; virtual bool periodic() const; virtual bool includesNorthPole() const; virtual bool includesSouthPole() const; }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Projection.h0000664000175000017500000000715615161702250017740 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/Point.h" #include "eckit/memory/Builder.h" #include "eckit/memory/Factory.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Generator.h" namespace eckit::geo { class Figure; } namespace eckit::geo { class Projection { public: // -- Types using builder_t = BuilderT1; using Spec = spec::Spec; using ARG1 = const Spec&; // -- Constructors explicit Projection(Figure* = nullptr); Projection(const Projection&) = default; Projection(Projection&&) = default; // -- Destructor virtual ~Projection() = default; // -- Operators Projection& operator=(const Projection&) = default; Projection& operator=(Projection&&) = default; // -- Methods virtual Point fwd(const Point&) const = 0; virtual Point inv(const Point&) const = 0; void falseXY(const PointXY& falseXY) { false_ = falseXY; } const PointXY& falseXY() const { return false_; } const Figure& figure() const; virtual void fill_spec(spec::Custom&) const; virtual const std::string& type() const = 0; [[nodiscard]] const Spec& spec() const; std::string spec_str() const { return spec().str(); } std::string proj_str() const; // -- Class methods static std::string className() { return "projection"; } static const Projection& projection_default(); [[nodiscard]] static Projection* make_from_spec(const Spec&); private: // -- Members mutable std::shared_ptr
figure_; mutable std::shared_ptr spec_; PointXY false_; // -- Friends friend class Grid; friend bool operator==(const Projection& a, const Projection& b) { return a.spec_str() == b.spec_str(); } friend bool operator!=(const Projection& a, const Projection& b) { return !(a == b); } }; using ProjectionFactoryType = Factory; using ProjectionSpecByName = spec::GeneratorT>; template using ProjectionRegisterType = ConcreteBuilderT1; template using ProjectionRegisterName = spec::ConcreteSpecGeneratorT1; struct ProjectionFactory { // This is 'const' as Projection should always be immutable [[nodiscard]] static const Projection* build(const Projection::Spec& spec) { return instance().make_from_spec_(spec); } // This is 'const' as Projection should always be immutable [[nodiscard]] static const Projection* make_from_string(const std::string&); [[nodiscard]] static Projection::Spec* make_spec(const Projection::Spec& spec) { return instance().make_spec_(spec); } static std::ostream& list(std::ostream& out) { return instance().list_(out); } static bool has_type(const std::string& type) { return ProjectionFactoryType::instance().exists(type); } private: static ProjectionFactory& instance(); // This is 'const' as Projection should always be immutable [[nodiscard]] const Projection* make_from_spec_(const Projection::Spec&) const; [[nodiscard]] Projection::Spec* make_spec_(const Projection::Spec&) const; std::ostream& list_(std::ostream&) const; }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/cache/0000775000175000017500000000000015161702250016505 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/cache/Record.h0000664000175000017500000000354715161702250020105 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/filesystem/PathName.h" #include "eckit/geo/cache/Download.h" #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/util/mutex.h" namespace eckit::spec { class Spec; } namespace eckit::geo::cache { std::string record_uid_from_spec(const spec::Spec&); std::string record_url_from_spec(const spec::Spec&); template class Record { public: using value_type = T; explicit Record(const PathName& cacheDir) : download_(cacheDir) {} const T& get(const spec::Spec& s) { return get(record_url_from_spec(s), record_uid_from_spec(s), ".ek", s); } const T& get(const std::string& url, const std::string& prefix, const std::string& suffix, const spec::Spec& spec) { util::lock_guard lock(mutex_); auto path = download_.to_cached_path(url, prefix, suffix); ASSERT_MSG(path.exists(), "Record: file '" + path + "' not found"); if (cache_.contains(path)) { return cache_[path]; } auto& record = cache_[path]; record.read(path); record.check(spec); return record; } void purge() { util::lock_guard lock(mutex_); cache_.purge(); } const PathName& cacheDir() const { return download_.cache_root(); } private: util::recursive_mutex mutex_; MemoryCacheT cache_; Download download_; }; } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/cache/MemoryCache.cc0000664000175000017500000000233015161702250021206 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/cache/MemoryCache.h" #include #include namespace eckit::geo::cache { static util::recursive_mutex MUTEX; static std::vector CACHES; MemoryCache::bytes_size_t MemoryCache::total_footprint() { util::lock_guard lock(MUTEX); return std::accumulate(CACHES.begin(), CACHES.end(), static_cast(0), [](bytes_size_t sum, const auto* cache) { return sum + cache->footprint(); }); } void MemoryCache::total_purge() { util::lock_guard lock(MUTEX); std::for_each(CACHES.begin(), CACHES.end(), [](auto* cache) { cache->purge(); }); } MemoryCache::MemoryCache() { util::lock_guard lock(MUTEX); CACHES.emplace_back(this); } } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/cache/Unzip.cc0000664000175000017500000001036515161702250020126 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/cache/Unzip.h" #include "eckit/eckit_config.h" #include "eckit/exception/Exceptions.h" #if eckit_HAVE_ZIP #include #include #include #include #include #include "eckit/os/AutoUmask.h" #include "eckit/utils/MD5.h" #include "zip.h" #endif namespace eckit::geo::cache { #if eckit_HAVE_ZIP using zip_type = std::unique_ptr; struct unzip_type : std::unique_ptr { explicit unzip_type(zip_file_t* ptr) : unique_ptr(ptr, zip_fclose) {} void to_path(const PathName& path) { // create directory/files with requested permissions, no masking AutoUmask umask(0); if (auto dir = path.dirName(); !dir.exists()) { dir.mkdir(); if (!dir.exists()) { throw WriteError("Unzip: error creating directory '" + dir + "'", Here()); } } // decompression buffer std::array buffer{ 0, }; std::ofstream out(path, std::ios::binary); for (zip_int64_t r = 0; 0 < (r = zip_fread(get(), buffer.data(), buffer.size()));) { out.write(buffer.data(), r); } if (!path.exists()) { throw WriteError("Unzip: error writing to '" + path + "'", Here()); } } }; void Unzip::to_path(const PathName& zip, const PathName& path, const std::string& what) { int errorp = 0; zip_type z(zip_open(zip.localPath(), ZIP_RDONLY, &errorp), zip_close); if (!z) { throw CantOpenFile(zip + "', errorp=" + std::to_string(errorp), Here()); } // Extract all files, or a specific one if (what.empty()) { const auto num_entries = static_cast(zip_get_num_entries(z.get(), 0)); for (zip_uint64_t i = 0; i < num_entries; ++i) { std::string what = zip_get_name(z.get(), i, 0); if (!what.empty() && what.back() != '/') { if (unzip_type unz(zip_fopen_index(z.get(), i, 0)); unz) { unz.to_path(path / what); } } } return; } if (unzip_type unz(zip_fopen(z.get(), what.c_str(), 0)); unz) { unz.to_path(path); return; } throw ReadError("Unzip: '" + what + "' not found in '" + zip + "'", Here()); } PathName Unzip::to_cached_path(const PathName& zip, const std::string& what, const std::string& prefix, const std::string& suffix) const { const auto key = MD5{zip / what}.digest(); const auto path = cache_root() / (zip.baseName() + ".dir") / (prefix + (prefix.empty() ? "" : "-") + key + suffix); if (!path.exists()) { to_path(zip, path, what); ASSERT_MSG(path.exists(), "Unzip: file '" + path + "' not found"); } return path; } std::vector Unzip::list(const PathName& zip, bool files_only) { std::vector list; int errorp = 0; zip_type z(zip_open(zip.localPath(), ZIP_RDONLY, &errorp), zip_close); if (!z) { throw CantOpenFile(zip + "', errorp=" + std::to_string(errorp), Here()); } const auto num_entries = static_cast(zip_get_num_entries(z.get(), 0)); list.reserve(num_entries); for (zip_uint64_t i = 0; i < num_entries; ++i) { if (std::string what = zip_get_name(z.get(), i, 0); !what.empty()) { if (!files_only || what.back() != '/') { list.emplace_back(what); } } } return list; } #else void Unzip::to_path(const PathName&, const PathName&, const std::string&) { NOTIMP; } PathName Unzip::to_cached_path(const PathName&, const std::string&, const std::string&, const std::string&) const { NOTIMP; } std::vector Unzip::list(const PathName&, bool) { NOTIMP; } #endif } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/cache/Download.cc0000664000175000017500000001115615161702250020567 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/cache/Download.h" #include "eckit/eckit_config.h" #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/util/mutex.h" #include "eckit/io/FileLock.h" #include "eckit/io/Length.h" #include "eckit/log/Log.h" #include "eckit/log/Timer.h" #include "eckit/os/AutoUmask.h" #include "eckit/utils/MD5.h" #if eckit_HAVE_CURL #include "eckit/io/URLHandle.h" #endif namespace eckit::geo::cache { static util::recursive_mutex MUTEX; class lock_type { util::lock_guard lock_guard_{MUTEX}; }; class file_lock_type { struct flock_type { explicit flock_type(const PathName& path) : lock_path_([](const auto& path) { AutoUmask umask(0); PathName lock(path + ".lock"); lock.touch(); return lock; }(path)), lock_(lock_path_) {} void lock() { lock_.lock(); } void unlock() { lock_.unlock(); lock_path_.unlink(false); } PathName lock_path_; FileLock lock_; }; flock_type flock_; util::lock_guard lock_guard_; public: explicit file_lock_type(const PathName& path) : flock_(path), lock_guard_(flock_) {} }; std::string Download::url_file_basename(const url_type& url, bool ext) { std::string n = url; // strip queries, directory and extension n = n.substr(0, n.find_first_of("?#")); if (auto f = n.find_last_of('/'); f != std::string::npos) { n = n.substr(f + 1); } if (auto f = n.find_last_of('.'); !ext && f != std::string::npos) { n = n.substr(0, f); } return n; } std::string Download::url_file_extension(const url_type& url) { // extension includes dot, e.g. ".jpg" auto n = url_file_basename(url); auto f = n.find_last_of('.'); return f != 0 && f != std::string::npos ? n.substr(f) : ""; } Download::info_type Download::to_path(const url_type& url, const PathName& path, bool html) { // control concurrent download lock_type lock; Length length = 0; Timer timer; #if eckit_HAVE_CURL // for eckit::URLHandle file_lock_type flock(path); auto tmp = path + ".part"; auto dir = path.dirName(); dir.mkdir(); ASSERT(dir.exists()); try { length = URLHandle{url}.saveInto(tmp); } catch (...) { length = 0; } // no empty files if (length <= 0) { if (tmp.exists()) { tmp.unlink(true); } throw UserError("Download error: '" + url + "' to '" + path + "'", Here()); } // no html response (eg. http standard response codes) in arbitrarily small files if (!html) { constexpr long long HTML_MAX_SIZE(32748); if (length < Length{HTML_MAX_SIZE}) { ASSERT(tmp.exists()); std::string contents{std::istreambuf_iterator(std::ifstream(tmp.asString()).rdbuf()), {}}; if (contents.find("") != std::string::npos) { tmp.unlink(true); throw UserError("Download error: '" + url + "' to '" + path + "'", Here()); } } } PathName::rename(tmp, path); #endif return {static_cast(length), timer.elapsed()}; } PathName Download::to_cached_path(const url_type& url, const std::string& prefix, const std::string& suffix) const { // control concurrent access lock_type lock; static MemoryCacheT CACHE; // set cache key, return path early if possible const auto key = MD5{url}.digest(); const auto path = CACHE.contains(key) ? PathName{CACHE[key]} : cache_root() / prefix + (prefix.empty() ? "" : "-") + key + suffix; if (path.exists()) { return CACHE[key] = path; } // download, update cache, return path Log::info() << "Downloading '" << url << "' to '" << path << "'..." << std::endl; auto info = Download::to_path(url, path, html_); Log::info() << "Download of " << info.bytes << " took " << info.time_s << "s." << std::endl; ASSERT_MSG(path.exists(), "Download: file '" + path + "' not found"); return CACHE[key] = path; } } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/cache/DiskCache.cc0000664000175000017500000000174115161702250020635 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/cache/DiskCache.h" #include "eckit/geo/util/mutex.h" namespace eckit::geo::cache { void DiskCache::rmdir(const PathName& p) const { // control concurrent access static util::recursive_mutex MUTEX; util::lock_guard lock_guard_{MUTEX}; if (!p.exists()) { return; } std::vector files; std::vector dirs; p.children(files, dirs); for (auto& f : files) { f.unlink(); } for (auto& d : dirs) { rmdir(d); } p.rmdir(); } } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/cache/Download.h0000664000175000017500000000252215161702250020426 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/cache/DiskCache.h" #include "eckit/log/Bytes.h" namespace eckit::geo::cache { class Download final : public DiskCache { public: // -- Types using url_type = std::string; struct info_type { Bytes bytes; double time_s; }; // -- Constructors explicit Download(const PathName& root = ".", bool html = false) : DiskCache(root), html_(html) {} // -- Methods PathName to_cached_path(const url_type&, const std::string& prefix = "", const std::string& suffix = ".download") const; // -- Class methods static info_type to_path(const url_type&, const PathName&, bool html = false); static std::string url_file_basename(const url_type&, bool ext = true); static std::string url_file_extension(const url_type&); private: void rmdir(const PathName&) const; bool html_; }; } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/cache/MemoryCache.h0000664000175000017500000000567615161702250021070 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/util/mutex.h" namespace eckit::geo::cache { class MemoryCache { public: using bytes_size_t = decltype(sizeof(int)); static bytes_size_t total_footprint(); static void total_purge(); protected: MemoryCache(); private: virtual bytes_size_t footprint() const = 0; virtual void purge() = 0; }; template class MemoryCacheT final : private MemoryCache { private: template using footprint_t = decltype(std::declval().footprint()); template > struct has_footprint : std::false_type {}; template struct has_footprint>> : std::true_type {}; public: using key_type = Key; using value_type = Value; MemoryCacheT() : mutex_(new util::recursive_mutex) { ASSERT(mutex_ != nullptr); } ~MemoryCacheT() { delete mutex_; } MemoryCacheT(const MemoryCacheT&) = delete; MemoryCacheT(MemoryCacheT&&) = delete; MemoryCacheT& operator=(const MemoryCacheT&) = delete; MemoryCacheT& operator=(MemoryCacheT&&) = delete; bool contains(const key_type& key) const { util::lock_guard lock(*mutex_); return container_.find(key) != container_.end(); } const value_type& operator[](const key_type& key) const { util::lock_guard lock(*mutex_); return container_[key]; } value_type& operator[](const key_type& key) { util::lock_guard lock(*mutex_); return container_[key]; } bytes_size_t footprint() const final { util::lock_guard lock(*mutex_); return std::accumulate(container_.begin(), container_.end(), static_cast(0), [](bytes_size_t sum, const auto& kv) { return sum + footprint_impl(kv.second, has_footprint{}); }); } void purge() final { container_.clear(); } private: static bytes_size_t footprint_impl(const value_type& v, std::true_type) { return v.footprint(); } static bytes_size_t footprint_impl(const value_type& v, std::false_type) { return v.size() * sizeof(typename value_type::value_type); } mutable std::map container_; util::recursive_mutex* mutex_; }; } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/cache/DiskCache.h0000664000175000017500000000157215161702250020501 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/filesystem/PathName.h" namespace eckit::geo::cache { class DiskCache { public: // -- Methods void rm_cache_root() const { rmdir(root_); } const PathName& cache_root() const { return root_; } protected: // -- Constructors explicit DiskCache(const PathName& root) : root_{root} {} // -- Methods void rmdir(const PathName&) const; private: // -- Members const PathName root_; }; } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/cache/Record.cc0000664000175000017500000000140415161702250020231 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/cache/Record.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/spec/Spec.h" namespace eckit::geo::cache { std::string record_uid_from_spec(const spec::Spec& s) { return s.get_string("uid", ""); } std::string record_url_from_spec(const spec::Spec& s) { return LibEcKitGeo::url(s.get_string("url")); } } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/cache/Unzip.h0000664000175000017500000000212715161702250017765 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/cache/DiskCache.h" namespace eckit::geo::cache { class Unzip final : public DiskCache { public: // -- Constructors explicit Unzip(const PathName& root = ".") : DiskCache(root) {} // -- Methods PathName to_cached_path(const PathName& zip, const std::string& what = {}, const std::string& prefix = "", const std::string& suffix = ".unzip") const; // -- Class methods static void to_path(const PathName& zip, const PathName&, const std::string& what = {}); static std::vector list(const PathName& zip, bool files_only = true); }; } // namespace eckit::geo::cache eckit-2.0.7/src/eckit/geo/Figure.h0000664000175000017500000000520515161702250017036 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/PointLonLat.h" #include "eckit/geo/PointXYZ.h" #include "eckit/memory/Builder.h" #include "eckit/memory/Factory.h" namespace eckit { namespace geo { namespace area { class BoundingBox; } class Projection; } // namespace geo namespace spec { class Custom; class Spec; } // namespace spec } // namespace eckit namespace eckit::geo { /** * @brief Figure: describe a combination of "shape" (sphere, ellipsoid, geoid) and "size" (radius, a, b, elevation) */ class Figure { public: // -- Types using builder_t = BuilderT0
; using Spec = spec::Spec; // -- Constructors Figure() noexcept = default; Figure(const Figure&) = delete; Figure(Figure&&) = delete; explicit Figure(const Spec&); // -- Destructor virtual ~Figure() = default; // -- Operators Figure& operator=(const Figure&) = delete; Figure& operator=(Figure&&) = delete; // -- Methods static std::string className() { return "figure"; } virtual double R() const; virtual double a() const; virtual double b() const; /// Surface area [L^2] virtual double area() const; /// Surface area between parallels and meridians [L^2] virtual double area(const area::BoundingBox&) const; [[nodiscard]] spec::Custom* spec() const; std::string spec_str() const; std::string proj_str() const; bool spherical() const; double eccentricity() const; double flattening() const; private: // -- Methods virtual void fill_spec(spec::Custom&) const; // -- Friends friend bool operator==(const Figure& a, const Figure& b) { return a.spec_str() == b.spec_str(); } friend bool operator!=(const Figure& a, const Figure& b) { return !(a == b); } friend class Projection; }; struct FigureFactory { [[nodiscard]] static Figure* build(const Figure::Spec& spec) { return instance().make_from_spec_(spec); } [[nodiscard]] static Figure* make_from_string(const std::string&); private: static FigureFactory& instance(); [[nodiscard]] Figure* make_from_spec_(const Figure::Spec&) const; }; template using FigureBuilder = ConcreteBuilderT0; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/PointLonLat.cc0000664000175000017500000001064515161702250020162 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/PointLonLat.h" #include #include #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/util.h" #include "eckit/spec/Spec.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo { PointLonLat::value_type PointLonLat::normalise_angle_to_minimum(value_type a, value_type minimum) { const auto modulus = [](auto a) { return a - FULL_ANGLE * std::floor(a / FULL_ANGLE); }; auto diff = a - minimum; return 0. <= diff && diff < FULL_ANGLE ? a : (modulus(diff) + minimum); } PointLonLat::value_type PointLonLat::normalise_angle_to_maximum(value_type a, value_type maximum) { const auto modulus = [](auto a) { return a - FULL_ANGLE * std::ceil(a / FULL_ANGLE); }; auto diff = a - maximum; return -FULL_ANGLE < diff && diff <= 0. ? a : (modulus(diff) + maximum); } bool PointLonLat::pole(value_type eps) const { const auto p = make(lon(), lat()); return types::is_approximately_equal(p.lat(), RIGHT_ANGLE, eps) || types::is_approximately_equal(p.lat(), -RIGHT_ANGLE, eps); } bool PointLonLat::north_pole(value_type eps) const { return types::is_approximately_equal(make(lon(), lat()).lat(), RIGHT_ANGLE, eps); } bool PointLonLat::south_pole(value_type eps) const { return types::is_approximately_equal(make(lon(), lat()).lat(), -RIGHT_ANGLE, eps); } void PointLonLat::assert_latitude_range(const PointLonLat& P) { if (!(-RIGHT_ANGLE <= P.lat() && P.lat() <= RIGHT_ANGLE)) { std::ostringstream oss; oss.precision(std::numeric_limits::max_digits10); oss << "Invalid latitude [degree] " << P.lat(); throw BadValue(oss.str(), Here()); } } PointLonLat PointLonLat::make(value_type lon, value_type lat, value_type lon_minimum, value_type eps) { lat = normalise_angle_to_minimum(lat, -RIGHT_ANGLE); if (types::is_strictly_greater(lat, RIGHT_ANGLE, eps)) { lat = FLAT_ANGLE - lat; lon += FLAT_ANGLE; } return types::is_approximately_equal(lat, RIGHT_ANGLE, eps) ? NORTH_POLE : types::is_approximately_equal(lat, -RIGHT_ANGLE, eps) ? SOUTH_POLE : PointLonLat{normalise_angle_to_minimum(lon, lon_minimum), lat}; } PointLonLat PointLonLat::make_from_lonlatr(value_type lonr, value_type latr, value_type lon_minimum) { return make(util::RADIAN_TO_DEGREE * lonr, util::RADIAN_TO_DEGREE * latr, lon_minimum); } PointLonLat PointLonLat::make_from_spec(const spec::Spec& spec, const std::string& name) { static const PointLonLat dfault; if (spec.has(name + "_lonlat") || (spec.has(name + "_lon") && spec.has(name + "_lat"))) { return make_from_spec(spec, name, dfault); } throw exception::SpecError( "PointLonLat::make_from_spec: missing '" + name + "_lonlat' or '" + name + "_lon'/'" + name + "_lat'", Here()); } PointLonLat PointLonLat::make_from_spec(const spec::Spec& spec, const std::string& name, const PointLonLat& dfault) { if (std::vector v{dfault.lon(), dfault.lat()}; (spec.get(name + "_lonlat", v) && v.size() == 2) || (spec.get(name + "_lon", v[0]) && spec.get(name + "_lat", v[1]))) { return {v[0], v[1]}; } return dfault; } PointLonLat PointLonLat::componentsMin(const PointLonLat& p, const PointLonLat& q) { return {std::min(p.lon(), q.lon()), std::min(p.lat(), q.lat())}; } PointLonLat PointLonLat::componentsMax(const PointLonLat& p, const PointLonLat& q) { return {std::max(p.lon(), q.lon()), std::max(p.lat(), q.lat())}; } bool points_equal(const PointLonLat& a, const PointLonLat& b, PointLonLat::value_type eps) { const auto c = PointLonLat::make(a.lon(), a.lat(), 0., eps); const auto d = PointLonLat::make(b.lon(), b.lat(), 0., eps); return types::is_approximately_equal(c.lon(), d.lon(), eps) && types::is_approximately_equal(c.lat(), d.lat(), eps); } const PointLonLat NORTH_POLE{0., PointLonLat::RIGHT_ANGLE}; const PointLonLat SOUTH_POLE{0., -PointLonLat::RIGHT_ANGLE}; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/PointLonLat.h0000664000175000017500000001050215161702250020014 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include namespace eckit { namespace geo { class PointLonLatR; } namespace spec { class Spec; } } // namespace eckit namespace eckit::geo { /** * @brief The PointLonLat class * @details A point on a geographic coordinate system, in (longitude, latitude) coordinates [degree]; They are fully * circular in space (also latitude), longitude is typically limited to [0, 360[ and latitude to [-90, 90], with * normalisation functions available, as well as conversion to and from radian-based coordinates. */ class PointLonLat final : protected std::array { public: // -- Types using container_type = array; using container_type::value_type; // -- Constructors PointLonLat() : PointLonLat(0., 0.) {} PointLonLat(value_type lon, value_type lat) : container_type{lon, lat} {} PointLonLat(const PointLonLat& other) : container_type(other) {} PointLonLat(PointLonLat&& other) : container_type(other) {} // -- Destructor ~PointLonLat() = default; // -- Operators PointLonLat& operator=(const PointLonLat& other) { container_type::operator=(other); return *this; } PointLonLat& operator=(PointLonLat&& other) { container_type::operator=(other); return *this; } // -- Members value_type lon() const { return container_type::operator[](0); } value_type lat() const { return container_type::operator[](1); } // -- Methods using container_type::data; bool pole(value_type eps = EPS) const; bool north_pole(value_type eps = EPS) const; bool south_pole(value_type eps = EPS) const; static size_t dimensions() { return DIMS; } static value_type normalise_angle_to_minimum(value_type, value_type minimum); static value_type normalise_angle_to_maximum(value_type, value_type maximum); static void assert_latitude_range(const PointLonLat&); [[nodiscard]] static PointLonLat make(value_type lon, value_type lat, value_type lon_minimum = 0., value_type eps = EPS); [[nodiscard]] static PointLonLat make_from_lonlatr(value_type lonr, value_type latr, value_type lon_minimum = 0.); [[nodiscard]] static PointLonLat make_from_spec(const eckit::spec::Spec&, const std::string& name); [[nodiscard]] static PointLonLat make_from_spec(const eckit::spec::Spec&, const std::string& name, const PointLonLat& dfault); PointLonLat antipode() const { return make(lon(), lat() + FLAT_ANGLE); } // -- Class members static constexpr size_t DIMS = 2; static constexpr value_type EPS = 1e-9; static constexpr value_type FULL_ANGLE = 360.; static constexpr value_type FLAT_ANGLE = 180.; static constexpr value_type RIGHT_ANGLE = 90.; // -- Class methods static PointLonLat componentsMin(const PointLonLat& p, const PointLonLat& q); static PointLonLat componentsMax(const PointLonLat& p, const PointLonLat& q); // -- Friends friend std::ostream& operator<<(std::ostream& out, const PointLonLat& p) { return out << '{' << p.lon() << ", " << p.lat() << '}'; } friend PointLonLat operator-(const PointLonLat& p, const PointLonLat& q) { return {p.lon() - q.lon(), p.lat() - q.lat()}; } friend PointLonLat operator+(const PointLonLat& p, const PointLonLat& q) { return {p.lon() + q.lon(), p.lat() + q.lat()}; } friend PointLonLat operator*(const PointLonLat& p, value_type d) { return {p.lon() * d, p.lat() * d}; } friend bool operator<(const PointLonLat& p, const PointLonLat& q) { return static_cast(p) < static_cast(q); } }; bool points_equal(const PointLonLat&, const PointLonLat&, PointLonLat::value_type eps = PointLonLat::EPS); extern const PointLonLat NORTH_POLE; extern const PointLonLat SOUTH_POLE; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Point.cc0000664000175000017500000000204515161702250017043 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Point.h" #include #include "eckit/geo/Exceptions.h" namespace eckit::geo { bool points_equal(const Point& p, const Point& q) { ASSERT(p.index() == q.index()); return std::visit([&](const auto& p, const auto& q) { return points_equal(p, q); }, p, q); } bool points_equal(const Point& p, const Point& q, double eps) { ASSERT(p.index() == q.index()); return std::visit([&](const auto& p, const auto& q) { return points_equal(p, q, eps); }, p, q); } std::ostream& operator<<(std::ostream& out, const Point& p) { std::visit([&](const auto& p) { out << p; }, p); return out; } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/projection/0000775000175000017500000000000015161702250017616 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/projection/LonLatToXYZ.h0000664000175000017500000000374715161702250022111 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Projection.h" #include "eckit/geo/figure/Earth.h" namespace eckit::geo::projection { /// Calculate coordinates of a point on a sphere or spheroid, in [x, y, z] class LonLatToXYZ : public Projection { public: // -- Constructors explicit LonLatToXYZ(Figure* = new figure::Earth); explicit LonLatToXYZ(double R); explicit LonLatToXYZ(double a, double b); explicit LonLatToXYZ(const Spec&); // -- Methods inline PointXYZ fwd(const PointLonLat& p) const { return (*impl_)(p); } inline PointLonLat inv(const PointXYZ& q) const { return (*impl_)(q); } // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return (*impl_)(std::get(p)); } inline Point inv(const Point& q) const override { return (*impl_)(std::get(q)); } protected: // -- Overridden methods void fill_spec(spec::Custom&) const override; private: // -- Types struct Implementation { Implementation() = default; virtual ~Implementation() = default; Implementation(const Implementation&) = delete; Implementation(Implementation&&) = delete; void operator=(const Implementation&) = delete; void operator=(Implementation&&) = delete; virtual PointXYZ operator()(const PointLonLat&) const = 0; virtual PointLonLat operator()(const PointXYZ&) const = 0; }; // -- Members std::unique_ptr impl_; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/LambertConformalConic.h0000664000175000017500000000362615161702250024201 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Projection.h" namespace eckit::geo::projection { /** * @brief LambertConformalConic projection * @ref Map Projections: A Working Manual, John P. Snyder (1987) * @ref Wolfram MathWorld (http://mathworld.wolfram.com/LambertConformalConicProjection.html) */ class LambertConformalConic : public Projection { public: // -- Constructors explicit LambertConformalConic(const Spec&); explicit LambertConformalConic(double lat_1, double lon_0 = 0., double lat_0 = 0., double lat_2 = 0., Figure* = nullptr); // -- Methods PointXY fwd(const PointLonLat&) const; PointLonLat inv(const PointXY&) const; // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return fwd(std::get(p)); } inline Point inv(const Point& q) const override { return inv(std::get(q)); } protected: // -- Overridden methods void fill_spec(spec::Custom&) const override; private: // -- Members /// Central meridian/longitude of natural origin, longitude of origin or longitude of false origin const double lon_0_; /// Latitude of natural origin, latitude of false origin or latitude of projection centre const double lat_0_; /// First standard parallel const double lat_1_; /// Second standard parallel const double lat_2_; double n_; double f_; double rho0_bare_; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/LonLatToXYZ.cc0000664000175000017500000000510615161702250022236 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/LonLatToXYZ.h" #include "eckit/geo/figure/OblateSpheroid.h" #include "eckit/geo/figure/Sphere.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Spec.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::projection { static const std::string TYPE("ll-to-xyz"); static ProjectionRegisterType PROJECTION(TYPE); LonLatToXYZ::LonLatToXYZ(Figure* figure_ptr) : Projection(figure_ptr) { struct LonLatToSphereXYZ final : Implementation { const double R; explicit LonLatToSphereXYZ(double _R) : R(_R) {} PointXYZ operator()(const PointLonLat& p) const override { return figure::Sphere::convertSphericalToCartesian(R, p, 0.); } PointLonLat operator()(const PointXYZ& q) const override { return figure::Sphere::convertCartesianToSpherical(R, q); } }; struct LonLatToSpheroidXYZ final : Implementation { const double a; const double b; explicit LonLatToSpheroidXYZ(double _a, double _b) : a(_a), b(_b) {} PointXYZ operator()(const PointLonLat& p) const override { return figure::OblateSpheroid::convertSphericalToCartesian(a, b, p, 0.); } PointLonLat operator()(const PointXYZ& q) const override { NOTIMP; } }; impl_.reset(types::is_approximately_equal(figure().eccentricity(), 0.) ? static_cast(new LonLatToSphereXYZ(figure().R())) : new LonLatToSpheroidXYZ(figure().a(), figure().b())); } LonLatToXYZ::LonLatToXYZ(double R) : LonLatToXYZ(R, R) {} LonLatToXYZ::LonLatToXYZ(double a, double b) : LonLatToXYZ(types::is_approximately_equal(a, b) ? static_cast(new geo::figure::Sphere(a)) : new geo::figure::OblateSpheroid(a, b)) {} LonLatToXYZ::LonLatToXYZ(const Spec& spec) : LonLatToXYZ(FigureFactory::build(spec)) {} const std::string& LonLatToXYZ::type() const { static const std::string type{TYPE}; return type; } void LonLatToXYZ::fill_spec(spec::Custom& custom) const { Projection::fill_spec(custom); custom.set("type", TYPE); } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/Composer.h0000664000175000017500000000333315161702250021560 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/Projection.h" namespace eckit::geo::projection { class Composer : public Projection, private std::deque { public: // -- Constructors using deque::deque; explicit Composer() = default; Composer(const Composer&) = delete; Composer(Composer&&) = delete; // -- Destructor ~Composer() override { for (auto* p : *this) { delete p; } } // -- Operators Composer& operator=(const Composer&) = delete; Composer& operator=(Composer&&) = delete; // -- Methods using deque::clear; using deque::emplace_back; using deque::emplace_front; using deque::empty; using deque::size; std::vector fwd_points(const Point&) const; std::vector inv_points(const Point&) const; // -- Overridden methods const std::string& type() const override; Point fwd(const Point&) const override; Point inv(const Point&) const override; // -- Class methods [[nodiscard]] static Projection* compose_back(Projection*, const Spec&); [[nodiscard]] static Projection* compose_front(const Spec&, Projection*); private: // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/None.h0000664000175000017500000000170215161702250020666 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Projection.h" namespace eckit::geo::projection { class None : public Projection { public: // -- Constructors explicit None() = default; explicit None(const Spec&) {} // -- Overridden methods inline Point fwd(const Point& p) const override { return p; } inline Point inv(const Point& q) const override { return q; } const std::string& type() const override; private: // -- Overridden methods void fill_spec(spec::Custom&) const override {} }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/AlbersEqualArea.cc0000664000175000017500000000647215161702250023127 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/AlbersEqualArea.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/Figure.h" #include "eckit/geo/util.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::projection { static const ProjectionRegisterType PROJECTION_1("aea"); static const ProjectionRegisterType PROJECTION_2("albers"); static inline double calculate_C(double lat_1, double n) { const auto lat_1r = lat_1 * util::DEGREE_TO_RADIAN; return std::cos(lat_1r) * std::cos(lat_1r) + 2. * n * std::sin(lat_1r); } static inline double calculate_n(double lat_1, double lat_2) { if (auto n = 0.5 * (std::sin(lat_1 * util::DEGREE_TO_RADIAN) / std::sin(lat_2 * util::DEGREE_TO_RADIAN)); !types::is_approximately_equal(n, 0.)) { return n; } throw exception::ProjectionError("AlbersEqualArea: cannot calculate n", Here()); } static inline double calculate_rho(double R, double n, double C, double phir) { return R / n * std::sqrt(C - 2. * n * std::sin(phir)); } AlbersEqualArea::AlbersEqualArea(const Spec& spec) : AlbersEqualArea(spec.get_double("lon_0"), spec.get_double("lat_0"), spec.get_double("lat_1"), spec.get_double("lat_2"), FigureFactory::build(spec)) {} AlbersEqualArea::AlbersEqualArea(double lon_0, double lat_0, double lat_1, double lat_2, Figure* _figure) : Projection(_figure), centre_(lon_0, lat_0), centre_r_(PointLonLatR::make_from_lonlat(lon_0, lat_0)), lat_1_(lat_1), lat_2_(lat_2), n_(calculate_n(lat_1, lat_2)), R_(figure().R()), C_(calculate_C(lat_1, n_)), rho0_(calculate_rho(R_, n_, C_, centre_r_.latr())) {} PointXY AlbersEqualArea::fwd(const PointLonLat& p) const { auto pr = PointLonLatR::make_from_lonlat(p.lon(), p.lat()); auto rho = calculate_rho(R_, n_, C_, pr.latr()); auto thetar = n_ * (pr.lonr() - centre_r_.lonr()); return PointXY{rho * std::sin(thetar), rho0_ - rho * std::cos(thetar)} + falseXY(); } PointLonLat AlbersEqualArea::inv(const PointXY& p) const { auto q = p - falseXY(); auto rho = std::sqrt(q.X() * q.X() + (rho0_ - q.Y()) * (rho0_ - q.Y())); auto thetar = std::atan2(q.X(), rho0_ - q.Y()); return PointLonLat::make_from_lonlatr(centre_r_.lonr() + thetar / n_, std::asin((C_ - (rho * rho * n_ * n_) / (R_ * R_)) / (2. * n_)), centre_.lon() - PointLonLat::FLAT_ANGLE); } const std::string& AlbersEqualArea::type() const { static const std::string type{"aea"}; return type; } void AlbersEqualArea::fill_spec(spec::Custom& custom) const { Projection::fill_spec(custom); custom.set("type", type()); custom.set("lon_0", centre_.lon()); custom.set("lat_0", centre_.lat()); custom.set("lat_1", lat_1_); custom.set("lat_2", lat_2_); } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/EquidistantCylindrical.cc0000664000175000017500000000573015161702250024602 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/EquidistantCylindrical.h" #include #include "eckit/geo/util.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::projection { static ProjectionRegisterType PROJECTION1("eqc"); static ProjectionRegisterType PROJECTION2("plate-carree"); EquidistantCylindrical::EquidistantCylindrical(const Spec& spec) : EquidistantCylindrical(spec.get_double("lat_ts", 0.), spec.get_double("lat_0", 0.)) {} EquidistantCylindrical::EquidistantCylindrical(double lat_ts, double lat_0) : lat_ts_(types::is_approximately_equal(lat_ts, 0., PointLonLat::EPS) ? 0. : lat_ts), lat_0_(lat_0) { auto cos_lat_ts = std::cos(util::DEGREE_TO_RADIAN * lat_ts_); ASSERT(!types::is_approximately_equal(0., cos_lat_ts)); struct General final : Implementation { explicit General(double cos_lat_ts, double lat_0) : cos_lat_ts_(cos_lat_ts), inv_cos_lat_ts_(1. / cos_lat_ts_), lat_0_(lat_0) {} PointXY fwd(const PointLonLat& p) const override { return {p.lon() * cos_lat_ts_, p.lat() - lat_0_}; } PointLonLat inv(const PointXY& q) const override { return {q.X() * inv_cos_lat_ts_, q.Y() + lat_0_}; } const double cos_lat_ts_; const double inv_cos_lat_ts_; const double lat_0_; }; struct Specific final : Implementation { explicit Specific(double lat_0) : lat_0_(lat_0) {} PointXY fwd(const PointLonLat& p) const override { return {p.lon(), p.lat() - lat_0_}; } PointLonLat inv(const PointXY& q) const override { return {q.X(), q.Y() + lat_0_}; } const double lat_0_; }; impl_.reset(types::is_approximately_equal(lat_ts_, 0., PointLonLat::EPS) ? static_cast(new Specific(lat_0_)) : new General{cos_lat_ts, lat_0_}); } const std::string& EquidistantCylindrical::type() const { static const std::string type{"eqc"}; return type; } void EquidistantCylindrical::fill_spec(spec::Custom& custom) const { // this is the default projection so spec is only set with non-default parameters if (auto set_lat_ts = !types::is_approximately_equal(lat_ts_, 0., PointLonLat::EPS), set_lat_0 = !types::is_approximately_equal(lat_0_, 0., PointLonLat::EPS); set_lat_ts || set_lat_0) { if (set_lat_ts) { custom.set("lat_ts", lat_ts_); } if (set_lat_0) { custom.set("lat_0", lat_0_); } custom.set("type", type()); } } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/Stretch.h0000664000175000017500000000256015161702250021406 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Projection.h" namespace eckit::geo::projection { class Stretch : public Projection { public: // -- Constructors explicit Stretch(double c); explicit Stretch(const Spec&); // -- Methods inline PointLonLat fwd(const PointLonLat& p) const { return PointLonLat::make(p.lon(), stretch(p.lat(), 1. / c_)); } inline PointLonLat inv(const PointLonLat& p) const { return PointLonLat::make(p.lon(), stretch(p.lat(), c_)); } // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return fwd(std::get(p)); } inline Point inv(const Point& q) const override { return inv(std::get(q)); } protected: // -- Overridden methods void fill_spec(spec::Custom&) const override; private: // -- Members double c_; // -- Methods static double stretch(double a, double c); }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/PolarStereographic.h0000664000175000017500000000302015161702250023557 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Projection.h" namespace eckit::geo::projection { class PolarStereographic : public Projection { public: // -- Constructors explicit PolarStereographic(const Spec&); PolarStereographic(PointLonLat centre, PointLonLat first, Figure* = nullptr); // -- Methods PointXY fwd(const PointLonLat&) const; PointLonLat inv(const PointXY&) const; // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return fwd(std::get(p)); } inline Point inv(const Point& q) const override { return inv(std::get(q)); } protected: // -- Overridden methods void fill_spec(spec::Custom&) const override; private: // -- Members const PointLonLat centre_; // projection centre [degree] const PointLonLatR centre_r_; // projection centre [radian] const PointLonLat first_; // first point [degree] const PointLonLatR first_r_; // first point [radian] const double sign_; const double F_; double x0_; double y0_; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/Mercator.h0000664000175000017500000000334415161702250021547 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Projection.h" namespace eckit::geo::projection { /// Calculate coordinates of a point on a rotated sphere given new location of South Pole (vector) and angle class Mercator : public Projection { public: // -- Constructors explicit Mercator(PointLonLat centre, PointLonLat first = {0, 0}, Figure* = nullptr); explicit Mercator(const Spec&); // -- Methods PointXY fwd(const PointLonLat&) const; PointLonLat inv(const PointXY&) const; // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return fwd(std::get(p)); } inline Point inv(const Point& q) const override { return inv(std::get(q)); } protected: // -- Overridden methods void fill_spec(spec::Custom&) const override; private: // -- Members const PointLonLat centre_; // angle [degree] between Eastward direction and the Equator, range [0, 90], latitude // [degree] of projection intersecting ellipsoid const PointLonLat first_; const double eps_; const size_t max_iter_; double lam0_; double x0_; double y0_; double e_; double m_; double w_; // -- Methods double calculate_phi(double t) const; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/LambertConformalConic.cc0000664000175000017500000000773215161702250024341 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/LambertConformalConic.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/Figure.h" #include "eckit/geo/util.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::projection { static const ProjectionRegisterType PROJECTION_1("lcc"); static const ProjectionRegisterType PROJECTION_2("lambert"); static const ProjectionRegisterType PROJECTION_3("lambert_lam"); LambertConformalConic::LambertConformalConic(const Spec& spec) : LambertConformalConic(spec.get_double("lat_1"), spec.get_double("lon_0", 0.), spec.get_double("lat_0", 0.), spec.get_double("lat_2", 0.), FigureFactory::build(spec)) {} LambertConformalConic::LambertConformalConic(double lat_1, double lon_0, double lat_0, double lat_2, Figure* fig) : Projection(fig), lon_0_(lon_0), lat_0_(lat_0), lat_1_(lat_1), lat_2_(lat_2) { ASSERT(types::is_strictly_greater(figure().R(), 0.)); auto lat_1r = lat_1 * util::DEGREE_TO_RADIAN; auto lat_2r = lat_2 * util::DEGREE_TO_RADIAN; if (types::is_approximately_equal(lat_1, -lat_2)) { throw exception::ProjectionError( "LambertConformalConic: cannot have standard parallels on opposite sides of equator", Here()); } n_ = types::is_approximately_equal(lat_1, lat_2) ? std::sin(lat_1r) : std::log(std::cos(lat_1r) / std::cos(lat_2r)) / std::log(std::tan(M_PI_4 + lat_2r / 2.) / std::tan(M_PI_4 + lat_1r / 2.)); if (types::is_approximately_equal(n_, 0.)) { throw exception::ProjectionError("LambertConformalConic: cannot calculate n", Here()); } f_ = (std::cos(lat_1r) * std::pow(std::tan(M_PI_4 + lat_1r / 2.), n_)) / n_; rho0_bare_ = f_ * std::pow(std::tan(M_PI_4 + lat_0_ * util::DEGREE_TO_RADIAN / 2.), -n_); } PointXY LambertConformalConic::fwd(const PointLonLat& p) const { auto q = PointLonLatR::make_from_lonlat(p.lon(), p.lat()); auto rho = figure().R() * f_ * std::pow(std::tan(M_PI_4 + q.latr() / 2.), -n_); auto rho0 = figure().R() * rho0_bare_; // scaled auto dlam = q.lonr() - lon_0_ * util::DEGREE_TO_RADIAN; return {rho * std::sin(n_ * dlam), rho0 - rho * std::cos(n_ * dlam)}; } PointLonLat LambertConformalConic::inv(const PointXY& p) const { auto x = p.X() / figure().R(); auto y = rho0_bare_ - p.Y() / figure().R(); if (auto rho = std::hypot(x, y); !types::is_approximately_equal(rho, 0.)) { if (n_ < 0.) { rho = -rho; x = -x; y = -y; } auto lonr = std::atan2(x, y) / n_ + lon_0_ * util::DEGREE_TO_RADIAN; auto latr = 2. * std::atan(std::pow(f_ / rho, 1. / n_)) - M_PI_2; return PointLonLat::make_from_lonlatr(lonr, latr, lon_0_ - PointLonLat::FLAT_ANGLE); } return PointLonLat::make(0., n_ > 0 ? PointLonLat::RIGHT_ANGLE : -PointLonLat::RIGHT_ANGLE); } const std::string& LambertConformalConic::type() const { static const std::string type{"lcc"}; return type; } void LambertConformalConic::fill_spec(spec::Custom& custom) const { Projection::fill_spec(custom); custom.set("type", type()); custom.set("lat_1", lat_1_); if (!types::is_approximately_equal(lon_0_, 0.)) { custom.set("lon_0", lon_0_); } if (!types::is_approximately_equal(lat_0_, 0.)) { custom.set("lat_0", lat_0_); }; if (!types::is_approximately_equal(lat_2_, 0.)) { custom.set("lat_2", lat_2_); } } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/EquidistantCylindrical.h0000664000175000017500000000406115161702250024440 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Projection.h" namespace eckit::geo::projection { /// Calculate coordinates of a point on a rotated sphere given new location of South Pole (vector) and angle class EquidistantCylindrical : public Projection { public: // -- Constructors explicit EquidistantCylindrical(const Spec&); explicit EquidistantCylindrical(double lat_ts = 0, double lat_0 = 0); // -- Methods inline PointXY fwd(const PointLonLat& p) const { return impl_->fwd(p); } inline PointLonLat inv(const PointXY& q) const { return impl_->inv(q); } // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return fwd(std::get(p)); } inline Point inv(const Point& q) const override { return inv(std::get(q)); } // -- Class methods [[nodiscard]] static EquidistantCylindrical* make_from_spec(const Spec&); protected: // -- Overridden methods void fill_spec(spec::Custom&) const override; private: // -- Types struct Implementation { Implementation() = default; virtual ~Implementation() = default; Implementation(const Implementation&) = delete; Implementation(Implementation&&) = delete; void operator=(const Implementation&) = delete; void operator=(Implementation&&) = delete; virtual PointXY fwd(const PointLonLat&) const = 0; virtual PointLonLat inv(const PointXY&) const = 0; }; // -- Members std::shared_ptr impl_; double lat_ts_; double lat_0_; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/Composer.cc0000664000175000017500000000426415161702250021722 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/Composer.h" #include "eckit/spec/Custom.h" namespace eckit::geo::projection { std::vector Composer::fwd_points(const Point& p) const { if (empty()) { return {}; } std::vector points; points.reserve(size()); auto q = p; for (const auto* proj : *this) { points.emplace_back(proj->fwd(q)); q = points.back(); } return points; } std::vector Composer::inv_points(const Point& p) const { if (empty()) { return {}; } std::vector points; points.reserve(size()); auto q = p; for (auto proj = rbegin(); proj != rend(); ++proj) { points.emplace_back((*proj)->inv(q)); q = points.back(); } return points; } const std::string& Composer::type() const { static const std::string type{"composer"}; return type; } void Composer::fill_spec(spec::Custom& custom) const { std::vector specs; for (const auto* proj : *this) { specs.emplace_back(proj->spec_str()); } custom.set("projections", specs); } Point Composer::fwd(const Point& p) const { auto q = p; for (const auto* proj : *this) { q = proj->fwd(q); } return q; } Point Composer::inv(const Point& p) const { auto q = p; for (auto proj = rbegin(); proj != rend(); ++proj) { q = (*proj)->inv(q); } return q; } Projection* Composer::compose_back(Projection* p, const Spec& spec) { return new Composer{p, ProjectionFactoryType::instance().get(spec.get_string("type")).create(spec)}; } Projection* Composer::compose_front(const Spec& spec, Projection* p) { return new Composer{ProjectionFactoryType::instance().get(spec.get_string("type")).create(spec), p}; } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/Stretch.cc0000664000175000017500000000272015161702250021542 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/Stretch.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/util.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::projection { static ProjectionRegisterType PROJECTION("stretch"); Stretch::Stretch(double c) : c_(c) { if (types::is_approximately_equal(c_, 0.)) { throw exception::ProjectionError("Stretch: stretching_factor != 0", Here()); } ASSERT(c_ != 0.); } Stretch::Stretch(const Spec& spec) : Stretch(spec.get_double("stretching_factor")) {} const std::string& Stretch::type() const { static const std::string type{"stretch"}; return type; } double Stretch::stretch(double a, double c) { auto ar = util::DEGREE_TO_RADIAN * a; ar = std::asin(std::cos(2. * std::atan(c * std::tan(std::acos(std::sin(ar)) * 0.5)))); return util::RADIAN_TO_DEGREE * ar; } void Stretch::fill_spec(spec::Custom& spec) const { spec.set("type", "stretch"); spec.set("stretching_factor", c_); } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/SpaceView.cc0000664000175000017500000000200715161702250022012 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/SpaceView.h" #include "eckit/geo/Exceptions.h" #include "eckit/spec/Custom.h" namespace eckit::geo::projection { SpaceView::SpaceView(const Spec&) { NOTIMP; } PointXY SpaceView::fwd(const PointLonLat&) const { NOTIMP; } PointLonLat SpaceView::inv(const PointXY&) const { NOTIMP; } const std::string& SpaceView::type() const { static const std::string type{"space-view"}; return type; } void SpaceView::fill_spec(spec::Custom& custom) const { Projection::fill_spec(custom); custom.set("type", "geos"); //? NOTIMP; } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/Reverse.h0000664000175000017500000000241215161702250021401 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Point.h" #include "eckit/spec/Custom.h" namespace eckit::geo::projection { /** * @brief Reverse class * @details Used to reverse the forward and inverse methods of a projection. */ template class Reverse : public P { public: // -- Constructors using P::P; // -- Overridden methods inline Point fwd(const Point& p) const override { return P::inv(p); } inline Point inv(const Point& p) const override { return P::fwd(p); } private: // -- Overridden methods void fill_spec(spec::Custom& custom) const override { P::fill_spec(custom); // spec is only set of non-default projection, or default projection with non-default parameters if (!custom.empty()) { custom.set("type", "reverse-" + custom.get_string("type")); } } }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/SpaceView.h0000664000175000017500000000227015161702250021656 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Projection.h" namespace eckit::geo::projection { /** * @brief SpaceView projection * @ref LRIT/HRIT Global Specification (CGMS 03, Issue 2.6, 12.08.1999) */ class SpaceView : public Projection { public: // -- Constructors explicit SpaceView(const Spec&); // -- Methods PointXY fwd(const PointLonLat&) const; PointLonLat inv(const PointXY&) const; // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return fwd(std::get(p)); } inline Point inv(const Point& q) const override { return inv(std::get(q)); } protected: // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/AlbersEqualArea.h0000664000175000017500000000304015161702250022755 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Projection.h" namespace eckit::geo::projection { /** * @brief AlbersEqualArea projection for the sphere */ class AlbersEqualArea : public Projection { public: // -- Constructors explicit AlbersEqualArea(const Spec&); AlbersEqualArea(double lon_0, double lat_0, double lat_1, double lat_2, Figure* = nullptr); // -- Methods PointXY fwd(const PointLonLat&) const; PointLonLat inv(const PointXY&) const; // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return fwd(std::get(p)); } inline Point inv(const Point& q) const override { return inv(std::get(q)); } private: // -- Members const PointLonLat centre_; // central meridian/parallel [degree] const PointLonLatR centre_r_; // central meridian/parallel [radian] const double lat_1_; const double lat_2_; const double n_; const double R_; const double C_; const double rho0_; // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/None.cc0000664000175000017500000000124415161702250021025 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/None.h" namespace eckit::geo::projection { static ProjectionRegisterType PROJECTION("none"); const std::string& None::type() const { static const std::string type{"none"}; return type; } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/Rotation.cc0000664000175000017500000001433615161702250021733 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/Rotation.h" #include #include "eckit/geo/figure/UnitSphere.h" #include "eckit/geo/util.h" #include "eckit/maths/Matrix3.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::projection { static ProjectionRegisterType PROJECTION("rotation"); Rotation::Rotation(const Spec& spec) : Rotation( [](const auto& spec) -> PointLonLat { if (std::vector r; spec.get("rotation", r)) { ASSERT_MSG(r.size() == 2, "Rotation: expected 'rotation' as a list of size 2"); return {r[0], r[1]}; } if (auto lon = SOUTH_POLE.lon(), lat = SOUTH_POLE.lat(); spec.get("south_pole_lon", lon) && spec.get("south_pole_lat", lat)) { return {lon, lat}; } return SOUTH_POLE; }(spec), [](const auto& spec) -> double { double angle = 0.; spec.get("rotation_angle", angle); return angle; }(spec)) {} Rotation::Rotation(const PointLonLat& south_pole, double angle) : south_pole_(south_pole), angle_(angle), rotated_(true) { using M = maths::Matrix3; struct NonRotated final : Implementation { PointLonLat operator()(const PointLonLat& p) const override { return p; } }; struct RotationAngle final : Implementation { explicit RotationAngle(double angle) : angle_(angle) {} PointLonLat operator()(const PointLonLat& p) const override { return {p.lon() + angle_, p.lat()}; } const double angle_; }; struct RotationMatrix final : Implementation { explicit RotationMatrix(M&& R) : R_(R) {} PointLonLat operator()(const PointLonLat& p) const override { return figure::UnitSphere::_convertCartesianToSpherical( R_ * figure::UnitSphere::_convertSphericalToCartesian(p)); } const M R_; }; const auto alpha = util::DEGREE_TO_RADIAN * angle; const auto theta = util::DEGREE_TO_RADIAN * -(south_pole_.lat() + 90.); const auto phi = util::DEGREE_TO_RADIAN * -south_pole_.lon(); const auto ca = std::cos(alpha); const auto ct = std::cos(theta); const auto cp = std::cos(phi); if (types::is_approximately_equal(ct, 1., PointLonLat::EPS * util::DEGREE_TO_RADIAN)) { angle_ = PointLonLat::normalise_angle_to_minimum(angle_ - south_pole_.lon(), -PointLonLat::FLAT_ANGLE); rotated_ = !types::is_approximately_equal(angle_, 0., PointLonLat::EPS); fwd_.reset(rotated_ ? static_cast(new RotationAngle(-angle)) : new NonRotated); inv_.reset(rotated_ ? static_cast(new RotationAngle(angle)) : new NonRotated); return; } // FIXME this supports either angle-based or matrix-based rotation (but not both); // Implementing as Euler angles rotation matrix (ideal, but reordering Rz Ry Ra) changes the existing unit test const auto sa = std::sin(alpha); const auto st = std::sin(theta); const auto sp = std::sin(phi); // Rotate: rotate by α, then ϑ (y-axis, along the rotated Greenwich meridian), then φ (z-axis) // q = Rz Ry Ra p = [ cosφ sinφ ] [ cosϑ sinϑ ] [ cosα sinα ] p // [ -sinφ cosφ ] [ 1 ] [ -sinα cosα ] // [ 1 ] [ -sinϑ cosϑ ] [ 1 ] fwd_ = std::make_shared(M{ca * cp * ct - sa * sp, sa * cp * ct + ca * sp, cp * st, // -sa * cp - ca * ct * sp, ca * cp - sa * ct * sp, -sp * st, // -ca * st, -sa * st, ct}); // Un-rotate (rotate by -φ, -ϑ, -α): // p = Ra Ry Rz q = [ cosα -sinα ] [ cosϑ -sinϑ ] [ cosφ -sinφ ] q // [ sinα cosα ] [ 1 ] [ sinφ cosφ ] // [ 1 ] [ sinϑ cosϑ ] [ 1 ] inv_ = std::make_shared(M{ca * cp * ct - sa * sp, -sa * cp - ca * ct * sp, -ca * st, // sa * cp * ct + ca * sp, ca * cp - sa * ct * sp, -sa * st, // cp * st, -sp * st, ct}); angle_ = PointLonLat::normalise_angle_to_minimum(angle_, -PointLonLat::FLAT_ANGLE); } const std::string& Rotation::type() const { static const std::string type{"rotation"}; return type; } Rotation* Rotation::make_from_spec(const Spec& spec) { double angle = 0.; spec.get("rotation_angle", angle); auto lon = SOUTH_POLE.lon(); auto lat = SOUTH_POLE.lat(); if (std::vector r{lon, lat}; spec.get("rotation", r)) { ASSERT_MSG(r.size() == 2, "Rotation: expected 'rotation' as a list of size 2"); lon = r[0]; lat = r[1]; } else { ASSERT_MSG(spec.get("south_pole_lon", lon) == spec.get("south_pole_lat", lat), "Rotation: expected 'south_pole_lon' and 'south_pole_lat'"); } auto* r = new Rotation{{lon, lat}, angle}; if (!r->rotated()) { delete r; r = nullptr; } return r; } void Rotation::fill_spec(spec::Custom& custom) const { bool projection = false; if (!points_equal(SOUTH_POLE, south_pole_)) { custom.set("south_pole_lon", south_pole_.lon()); custom.set("south_pole_lat", south_pole_.lat()); projection = true; } if (!types::is_approximately_equal(angle_, 0., PointLonLat::EPS)) { custom.set("rotation_angle", angle_); projection = true; } if (projection) { custom.set("type", type()); } } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/PROJ.cc0000664000175000017500000001723215161702250020704 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/PROJ.h" #include #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/Figure.h" #include "eckit/spec/Custom.h" namespace eckit::geo::projection { static ProjectionRegisterType PROJECTION("proj"); namespace { constexpr auto CTX = PJ_DEFAULT_CTX; static const std::string DEFAULT = "EPSG:4326"; // WGS84, latitude/longitude coordinate system struct pj_t : std::unique_ptr { explicit pj_t(element_type* ptr) : unique_ptr(ptr, &proj_destroy) {} }; struct ctx_t : std::unique_ptr { explicit ctx_t(element_type* ptr) : unique_ptr(ptr, &proj_context_destroy) {} }; struct Convert { Convert() = default; virtual ~Convert() = default; Convert(const Convert&) = delete; Convert(Convert&&) = delete; void operator=(const Convert&) = delete; void operator=(Convert&&) = delete; virtual PJ_COORD to_coord(const Point&) const = 0; virtual Point to_point(const PJ_COORD&) const = 0; }; struct LonLat final : Convert { PJ_COORD to_coord(const Point& p) const final { const auto& q = std::get(p); return proj_coord(q.lon(), q.lat(), 0, 0); } Point to_point(const PJ_COORD& c) const final { return PointLonLat::make(c.enu.e, c.enu.n, lon_minimum_); } explicit LonLat(double lon_minimum) : lon_minimum_(lon_minimum) {} const double lon_minimum_; }; struct XY final : Convert { PJ_COORD to_coord(const Point& p) const final { const auto& q = std::get(p); return proj_coord(q.X(), q.Y(), 0, 0); } Point to_point(const PJ_COORD& c) const final { return PointXY{c.xy.x, c.xy.y}; } }; struct XYZ final : Convert { PJ_COORD to_coord(const Point& p) const final { const auto& q = std::get(p); return proj_coord(q.X(), q.Y(), q.Z(), 0); } Point to_point(const PJ_COORD& c) const final { return PointXYZ{c.xy.x, c.xy.y, c.xyz.z}; } }; Figure* make_figure(const std::string& proj_str) { pj_t identity(proj_create_crs_to_crs(CTX, proj_str.c_str(), proj_str.c_str(), nullptr)); pj_t crs(proj_get_target_crs(CTX, identity.get())); pj_t ellipsoid(proj_get_ellipsoid(CTX, crs.get())); ASSERT(ellipsoid); double a = 0; double b = 0; ASSERT(proj_ellipsoid_get_parameters(CTX, ellipsoid.get(), &a, &b, nullptr, nullptr)); ASSERT(0 < b && b <= a); return FigureFactory::build(spec::Custom{{{"a", a}, {"b", b}}}); } } // namespace struct PROJ::Implementation { Implementation(PJ* pj_ptr, PJ_CONTEXT* pjc_ptr, Convert* source_ptr, Convert* target_ptr) : proj_(pj_ptr), ctx_(pjc_ptr), source_(source_ptr), target_(target_ptr) { ASSERT(proj_); ASSERT(source_); ASSERT(target_); } inline Point fwd(const Point& p) const { return target_->to_point(proj_trans(proj_.get(), PJ_FWD, source_->to_coord(p))); } inline Point inv(const Point& p) const { return source_->to_point(proj_trans(proj_.get(), PJ_INV, target_->to_coord(p))); } private: const pj_t proj_; const ctx_t ctx_; const std::unique_ptr source_; const std::unique_ptr target_; }; PROJ::PROJ(const std::string& source, const std::string& target, double lon_minimum) : Projection(make_figure(target)), source_(source), target_(target) { ASSERT(!source_.empty()); ASSERT(!target_.empty()); auto make_convert = [lon_minimum](const std::string& string) -> Convert* { pj_t identity(proj_create_crs_to_crs(CTX, string.c_str(), string.c_str(), nullptr)); pj_t crs(proj_get_target_crs(CTX, identity.get())); pj_t cs(proj_crs_get_coordinate_system(CTX, crs.get())); ASSERT(cs); auto type = proj_cs_get_type(CTX, cs.get()); auto dim = proj_cs_get_axis_count(CTX, cs.get()); return type == PJ_CS_TYPE_CARTESIAN && dim == 3 ? static_cast(new XYZ) : type == PJ_CS_TYPE_CARTESIAN && dim == 2 ? static_cast(new XY) : type == PJ_CS_TYPE_ELLIPSOIDAL ? static_cast(new LonLat(lon_minimum)) : type == PJ_CS_TYPE_SPHERICAL ? static_cast(new LonLat(lon_minimum)) : NOTIMP; }; // projection, normalised auto ctx = PJ_DEFAULT_CTX; implementation_ = std::make_unique( proj_normalize_for_visualization(ctx, proj_create_crs_to_crs(ctx, source_.c_str(), target_.c_str(), nullptr)), ctx, make_convert(source_), make_convert(target_)); ASSERT(implementation_); } PROJ::PROJ(const Spec& spec) : PROJ(spec.get_string("source", DEFAULT), spec.get_string("target", spec.get_string("proj", DEFAULT)), spec.get_double("lon_minimum", 0)) {} const std::string& PROJ::type() const { static const std::string type{"proj"}; return type; } Point PROJ::fwd(const Point& p) const { return implementation_->fwd(p); } Point PROJ::inv(const Point& q) const { return implementation_->inv(q); } std::string PROJ::proj_str(const spec::Custom& custom) { using key_value_type = std::pair; using keys_type = std::vector; struct key_value_compare { bool operator()(const key_value_type& a, const key_value_type& b) const { if (a.first != b.first) { // keys that come first in string for (const auto& key : keys_type{"proj"}) { if (a.first == key || b.first == key) { return a.first == key; } } // keys that come last in string for (const auto& key : keys_type{"R", "a", "b"}) { if (a.first == key || b.first == key) { return b.first == key; } } } return a < b; }; }; static const std::map KEYS{ {"type", "proj"}, {"figure", "ellps"}, {"r", "R"}, }; static const std::map VALUES{ {"mercator", "merc"}, {"reverse_mercator", "merc"}, {"grs80", "GRS80"}, {"wgs84", "WGS84"}, }; auto rename = [](const std::map& map, const std::string& key) { const auto it = map.find(key); return it != map.end() ? it->second : key; }; std::set set; for (const auto& [k, v] : custom.container()) { if (const auto& key = rename(KEYS, k); !key.empty()) { const auto& value = rename(VALUES, to_string(v)); set.emplace(key, value); } } std::string str; const auto* sep = "+"; for (const auto& [key, value] : set) { str += sep + key + "=" + value; sep = " +"; } return str; } void PROJ::fill_spec(spec::Custom& custom) const { custom.set("type", "proj"); if (source_ != DEFAULT) { custom.set("source", source_); } if (target_ != DEFAULT) { custom.set("target", target_); } } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/PROJ.h0000664000175000017500000000262115161702250020542 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Projection.h" namespace eckit::geo::projection { /// Calculate coordinates using PROJ class PROJ : public Projection { public: // -- Constructors PROJ(const std::string& source, const std::string& target, double lon_minimum = 0.); explicit PROJ(const Spec&); // -- Methods const std::string& source() const { return source_; } const std::string& target() const { return target_; } // -- Overridden methods const std::string& type() const override; Point fwd(const Point&) const override; Point inv(const Point&) const override; // -- Class methods static std::string proj_str(const spec::Custom&); private: // -- Types struct Implementation; // -- Members std::unique_ptr implementation_; const std::string source_; const std::string target_; // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/Mercator.cc0000664000175000017500000001041415161702250021701 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/Mercator.h" #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/Figure.h" #include "eckit/geo/util.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::projection { static ProjectionRegisterType PROJECTION_1("mercator"); static ProjectionRegisterType PROJECTION_2("merc"); Mercator::Mercator(PointLonLat centre, PointLonLat first, Figure* figure_ptr) : Projection(figure_ptr), centre_(PointLonLat::make(centre.lon(), centre.lat(), -PointLonLat::FLAT_ANGLE)), first_(first), eps_(1e-10), max_iter_(15) { // Map Projections: A Working Manual, John P. Snyder (1987) // - Equation (7-9) to calculate phi iteratively // - Equation (15-11) to calculate t if (types::is_approximately_equal(first.lat(), PointLonLat::RIGHT_ANGLE) || types::is_approximately_equal(first.lat(), -PointLonLat::RIGHT_ANGLE)) { throw exception::ProjectionError("Mercator: projection cannot be calculated at the poles", Here()); } auto lam0 = util::DEGREE_TO_RADIAN * centre_.lon(); auto phi0 = util::DEGREE_TO_RADIAN * centre_.lat(); auto lam1 = util::DEGREE_TO_RADIAN * first.lon(); auto phi1 = util::DEGREE_TO_RADIAN * first.lat(); e_ = figure().eccentricity(); lam0_ = lam0; m_ = figure().a() * std::cos(phi0) / (std::sqrt(1. - e_ * e_ * std::sin(phi0) * std::sin(phi0))); ASSERT(!types::is_approximately_equal(m_, 0.)); w_ = 1. / m_; x0_ = m_ * (lam0_ - lam1); y0_ = m_ * std::log(std::tan(M_PI_4 - 0.5 * phi1) / std::pow(((1. - e_ * std::sin(phi1)) / (1. + e_ * std::sin(phi1))), 0.5 * e_)); ASSERT(types::is_approximately_equal(phi1, calculate_phi(std::exp(y0_ * w_)), eps_)); } Mercator::Mercator(const Spec& spec) : Mercator({spec.get_double("lon_0"), spec.get_double("lat_ts")}, {spec.get_double("first_lon"), spec.get_double("first_lat")}, FigureFactory::build(spec)) {} double Mercator::calculate_phi(double t) const { auto phi = M_PI_2 - 2 * std::atan(t); for (size_t i = 0; i <= max_iter_; i++) { auto es = e_ * std::sin(phi); auto dphi = M_PI_2 - 2 * std::atan(t * (std::pow(((1. - es * es) / (1. + es * es)), 0.5 * e_))) - phi; phi += dphi; if (types::is_approximately_equal(dphi, 0., eps_)) { return phi; } } throw SeriousBug("Mercator: phi calculation failed to converge", Here()); } PointXY Mercator::fwd(const PointLonLat& p) const { auto phi = util::DEGREE_TO_RADIAN * p.lat(); auto lam = util::DEGREE_TO_RADIAN * p.lon(); auto s = std::sin(phi); return {x0_ + m_ * (lam - lam0_), types::is_approximately_equal(s, 1.) ? std::numeric_limits::infinity() : types::is_approximately_equal(s, -1.) ? -std::numeric_limits::infinity() : y0_ - m_ * std::log(std::tan(M_PI_4 - 0.5 * phi) / std::pow(((1. - e_ * s) / (1. + e_ * s)), 0.5 * e_))}; } PointLonLat Mercator::inv(const PointXY& q) const { return PointLonLat::make(util::RADIAN_TO_DEGREE * (lam0_ + (q.X() - x0_) * w_), util::RADIAN_TO_DEGREE * calculate_phi(std::exp(-(q.Y() - y0_) * w_))); } const std::string& Mercator::type() const { static const std::string type{"mercator"}; return type; } void Mercator::fill_spec(spec::Custom& custom) const { Projection::fill_spec(custom); custom.set("type", "mercator"); if (!types::is_approximately_equal(centre_.lat(), 0.)) { custom.set("lat_ts", centre_.lat()); } if (!types::is_approximately_equal(centre_.lon(), 0.)) { custom.set("lon_0", centre_.lon()); } } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/LambertAzimuthalEqualArea.cc0000664000175000017500000000514315161702250025156 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/LambertAzimuthalEqualArea.h" #include #include "eckit/geo/Figure.h" #include "eckit/spec/Custom.h" namespace eckit::geo::projection { static ProjectionRegisterType PROJECTION_1("laea"); static ProjectionRegisterType PROJECTION_2("lambert_azimuthal_equal_area"); LambertAzimuthalEqualArea::LambertAzimuthalEqualArea(const Spec& spec) : LambertAzimuthalEqualArea(PointLonLat::make_from_spec(spec, "centre"), PointLonLat::make_from_spec(spec, "first")) { } LambertAzimuthalEqualArea::LambertAzimuthalEqualArea(PointLonLat centre, PointLonLat first, Figure* figure_ptr) : Projection(figure_ptr), centre_(centre), centre_r_(PointLonLatR::make_from_lonlat(centre.lon(), centre.lat())), first_(first), first_r_(PointLonLatR::make_from_lonlat(first.lon(), first.lat())), phi0_(centre_r_.latr()), phi_(first_r_.latr()), dlam_(first_r_.lonr() - centre_r_.lonr()) {} PointXY LambertAzimuthalEqualArea::fwd(const PointLonLat& p) const { const auto kp = figure().R() * std::sqrt(2. / (1. + phi0_.sin * phi_.sin + phi0_.cos * phi_.cos * dlam_.cos)); auto x = kp * phi_.cos * dlam_.sin; auto y = kp * (phi0_.cos * phi_.sin - phi0_.sin * phi_.cos * dlam_.cos); return {x, y}; } PointLonLat LambertAzimuthalEqualArea::inv(const PointXY& p) const { auto rho = std::sqrt(p.X() * p.X() + p.Y() * p.Y()); const util::sincos_t c(2. * std::asin(rho / (2. * figure().R()))); return PointLonLat::make_from_lonlatr( centre_r_.lonr() + std::atan2(p.X() * c.sin, rho * phi0_.cos * c.cos - p.Y() * phi0_.sin * c.sin), std::asin(c.cos * phi0_.sin + p.Y() * c.sin * phi0_.cos / rho), centre_.lon() - PointLonLat::FLAT_ANGLE); } const std::string& LambertAzimuthalEqualArea::type() const { static const std::string type{"laea"}; return type; } void LambertAzimuthalEqualArea::fill_spec(spec::Custom& custom) const { Projection::fill_spec(custom); custom.set("type", type()); custom.set("centre_lonlat", std::vector{centre_.lon(), centre_.lat()}); custom.set("first_lonlat", std::vector{first_.lon(), first_.lat()}); } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/LambertAzimuthalEqualArea.h0000664000175000017500000000317415161702250025022 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Projection.h" #include "eckit/geo/util/sincos.h" namespace eckit::geo::projection { class LambertAzimuthalEqualArea : public Projection { public: // -- Constructors explicit LambertAzimuthalEqualArea(const Spec&); LambertAzimuthalEqualArea(PointLonLat centre, PointLonLat first, Figure* = nullptr); // -- Methods PointXY fwd(const PointLonLat&) const; PointLonLat inv(const PointXY&) const; // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return fwd(std::get(p)); } inline Point inv(const Point& q) const override { return inv(std::get(q)); } protected: // -- Overridden methods void fill_spec(spec::Custom&) const override; private: // -- Members const PointLonLat centre_; // central meridian/standard parallel [degree] const PointLonLatR centre_r_; // central meridian/standard parallel [radian] const PointLonLat first_; // first point [degree] const PointLonLatR first_r_; // first point [radian] const util::sincos_t phi0_; const util::sincos_t phi_; const util::sincos_t dlam_; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/Rotation.h0000664000175000017500000000427015161702250021571 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Projection.h" namespace eckit::geo::projection { /// Calculate coordinates of a point on a rotated sphere given new location of South Pole (vector) and angle class Rotation : public Projection { public: // -- Constructors explicit Rotation(const Spec&); explicit Rotation(const PointLonLat& = SOUTH_POLE, double angle = 0); // -- Methods PointLonLat south_pole() const { return south_pole_; } double angle() const { return angle_; } bool rotated() const { return rotated_; } inline PointLonLat fwd(const PointLonLat& p) const { return (*fwd_)(p); } inline PointLonLat inv(const PointLonLat& q) const { return (*inv_)(q); } // -- Overridden methods const std::string& type() const override; inline Point fwd(const Point& p) const override { return fwd(std::get(p)); } inline Point inv(const Point& q) const override { return inv(std::get(q)); } // -- Class methods [[nodiscard]] static Rotation* make_from_spec(const Spec&); protected: // -- Overridden methods void fill_spec(spec::Custom&) const override; private: // -- Types struct Implementation { Implementation() = default; virtual ~Implementation() = default; Implementation(const Implementation&) = delete; Implementation(Implementation&&) = delete; void operator=(const Implementation&) = delete; void operator=(Implementation&&) = delete; virtual PointLonLat operator()(const PointLonLat&) const = 0; }; // -- Members std::shared_ptr fwd_; std::shared_ptr inv_; PointLonLat south_pole_; double angle_; bool rotated_; }; } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/projection/PolarStereographic.cc0000664000175000017500000000533315161702250023726 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/projection/PolarStereographic.h" #include #include "eckit/geo/Figure.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::projection { PolarStereographic::PolarStereographic(const Spec& spec) : PolarStereographic(PointLonLat::make_from_spec(spec, "centre"), PointLonLat::make_from_spec(spec, "first")) {} PolarStereographic::PolarStereographic(PointLonLat centre, PointLonLat first, Figure* figure_ptr) : Projection(figure_ptr), centre_(centre), centre_r_(PointLonLatR::make_from_lonlat(centre.lon(), centre.lat())), first_(first), first_r_(PointLonLatR::make_from_lonlat(first.lon(), first.lat())), sign_(centre_.lat() < 0. ? -1. : 1.), F_(types::is_approximately_equal(centre_.lat(), PointLonLat::RIGHT_ANGLE, PointLonLat::EPS) || types::is_approximately_equal(centre_.lat(), -PointLonLat::RIGHT_ANGLE, PointLonLat::EPS) ? 0.5 : std::tan(0.5 * (M_PI_2 - sign_ * centre_r_.latr())) / std::cos(sign_ * centre_r_.latr())) { auto z = fwd(first_); x0_ = z.X(); y0_ = z.Y(); } PointXY PolarStereographic::fwd(const PointLonLat& q) const { auto p = PointLonLatR::make_from_lonlat(q.lon(), q.lat()); auto a = sign_ * (p.lonr() - centre_r_.lonr()); auto tsf = std::tan(0.5 * (M_PI_2 - sign_ * p.latr())); auto height = figure().R() * tsf / F_; return {-sign_ * height * std::sin(a), sign_ * height * std::cos(a)}; } PointLonLat PolarStereographic::inv(const PointXY& q) const { PointXY p{q.X() - x0_, q.Y() - y0_}; auto rh = std::sqrt(p.X() * p.X() + p.Y() * p.Y()); auto tsi = rh / figure().R() * F_; return PointLonLat::make_from_lonlatr(sign_ * std::atan2(sign_ * p.X(), -sign_ * p.Y()) + centre_r_.lonr(), sign_ * (M_PI_2 - 2 * std::atan(tsi))); } const std::string& PolarStereographic::type() const { static const std::string type{"polar-stereographic"}; return type; } void PolarStereographic::fill_spec(spec::Custom& custom) const { Projection::fill_spec(custom); custom.set("type", "stere"); custom.set("lon_0", centre_.lon()); custom.set("lat_0", centre_.lat()); custom.set("lon_first", first_.lon()); custom.set("lat_first", first_.lat()); } } // namespace eckit::geo::projection eckit-2.0.7/src/eckit/geo/iterator/0000775000175000017500000000000015161702250017273 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/iterator/Unstructured.cc0000664000175000017500000000452215161702250022314 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/iterator/Unstructured.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/container/PointsContainer.h" #include "eckit/geo/grid/Unstructured.h" namespace eckit::geo::iterator { struct Instance { explicit Instance(const Iterator::Spec& spec) : grid(dynamic_cast(GridFactory::build(spec))) { ASSERT(grid); } std::unique_ptr grid; }; struct UnstructuredInstance : Instance, Unstructured { explicit UnstructuredInstance(const Iterator::Spec& spec) : Instance(spec), Unstructured(*grid) {} }; Unstructured::Unstructured(const Grid& grid, size_t index, std::shared_ptr container) : container_(container), index_(index), size_(container_->size()), uid_(grid.uid()) { ASSERT(container_->size() == grid.size()); } Unstructured::Unstructured(const Grid& grid) : index_(grid.size()), size_(grid.size()), uid_(grid.uid()) {} bool Unstructured::operator==(const geo::Iterator& other) const { const auto* another = dynamic_cast(&other); return another != nullptr && index_ == another->index_ && uid_ == another->uid_; } bool Unstructured::operator++() { if (index_++; index_ < size_) { return true; } index_ = size_; // ensure it's invalid return false; } bool Unstructured::operator+=(difference_type d) { if (auto di = static_cast(index_); 0 <= di + d && di + d < static_cast(size_)) { index_ = static_cast(di + d); return true; } index_ = size_; // ensure it's invalid return false; } Unstructured::operator bool() const { return index_ < size_; } Point Unstructured::operator*() const { ASSERT(container_); return container_->get(index_); } static const IteratorRegisterType ITERATOR_TYPE("unstructured"); } // namespace eckit::geo::iterator eckit-2.0.7/src/eckit/geo/iterator/Unstructured.h0000664000175000017500000000236715161702250022163 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Iterator.h" namespace eckit::geo::container { class PointsContainer; } namespace eckit::geo::iterator { class Unstructured : public Iterator { public: // -- Constructors Unstructured(const Grid&, size_t index, std::shared_ptr); explicit Unstructured(const Grid&); private: // -- Members std::shared_ptr container_; size_t index_; const size_t size_; const std::string uid_; // -- Overridden methods bool operator==(const geo::Iterator&) const override; bool operator++() override; bool operator+=(difference_type) override; explicit operator bool() const override; Point operator*() const override; size_t index() const override { return index_; } }; } // namespace eckit::geo::iterator eckit-2.0.7/src/eckit/geo/iterator/Reduced.cc0000664000175000017500000000572015161702250021161 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/iterator/Reduced.h" #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/grid/Reduced.h" namespace eckit::geo::iterator { struct Instance { explicit Instance(const Iterator::Spec& spec) : grid(dynamic_cast(GridFactory::build(spec))) { ASSERT(grid); } std::unique_ptr grid; }; struct ReducedInstance : Instance, Reduced { explicit ReducedInstance(const Iterator::Spec& spec) : Instance(spec), Reduced(*grid) {} }; Reduced::Reduced(const Grid& grid, size_t index) : grid_(dynamic_cast(grid)), projection_(grid_.projection()), latitudes_(grid_.latitudes()), niacc_(grid_.nxacc()), size_(grid.size()), j_(0), index_(index) { if (index_ < size_) { longitudes_j_ = grid_.longitudes(j_ = j(index_)); ASSERT(niacc_[j_] <= index && index_ < niacc_[j_ + 1]); ASSERT(latitudes_.size() == grid_.ny()); } } bool Reduced::operator==(const Iterator& other) const { const auto* another = dynamic_cast(&other); return another != nullptr && index_ == another->index_; } bool Reduced::operator++() { if (index_++; index_ < size_) { if (!(index_ < niacc_[j_ + 1])) { longitudes_j_ = grid_.longitudes(++j_); } ASSERT(niacc_[j_] <= index_ && index_ < niacc_[j_ + 1]); return true; } index_ = size_; // ensure it's invalid return false; } bool Reduced::operator+=(difference_type d) { if (auto di = static_cast(index_); 0 <= di + d && di + d < static_cast(size_)) { if (index_ = static_cast(di + d); !(niacc_[j_] <= index_ && index_ < niacc_[j_ + 1])) { longitudes_j_ = grid_.longitudes(j_ = j(index_)); } ASSERT(niacc_[j_] <= index_ && index_ < niacc_[j_ + 1]); return true; } index_ = size_; // ensure it's invalid return false; } Reduced::operator bool() const { return index_ < size_; } Point Reduced::operator*() const { return projection_.fwd(PointXY{longitudes_j_.at(index_ - niacc_[j_]), latitudes_.at(j_)}); } size_t Reduced::j(size_t idx) const { ASSERT(idx < size_); auto dist = std::distance(niacc_.begin(), std::upper_bound(niacc_.begin(), niacc_.end(), idx)); ASSERT(1 <= dist && dist <= niacc_.size() - 1); return static_cast(dist - 1); } static const IteratorRegisterType ITERATOR_TYPE("reduced"); } // namespace eckit::geo::iterator eckit-2.0.7/src/eckit/geo/iterator/Reduced.h0000664000175000017500000000260315161702250021020 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Iterator.h" namespace eckit::geo { class Projection; namespace grid { class Reduced; } } // namespace eckit::geo namespace eckit::geo::iterator { class Reduced : public geo::Iterator { public: // -- Constructors explicit Reduced(const Grid&, size_t index = 0); private: // -- Members const grid::Reduced& grid_; const Projection& projection_; std::vector longitudes_j_; const std::vector& latitudes_; const std::vector& niacc_; const size_t size_; size_t j_; size_t index_; // -- Overridden operators bool operator==(const Iterator&) const override; bool operator++() override; bool operator+=(difference_type) override; explicit operator bool() const override; Point operator*() const override; // -- Overridden methods size_t index() const override { return index_; } size_t j(size_t idx) const; }; } // namespace eckit::geo::iterator eckit-2.0.7/src/eckit/geo/iterator/Regular.h0000664000175000017500000000252215161702250021046 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Iterator.h" namespace eckit::geo { class Projection; namespace grid { class Regular; } } // namespace eckit::geo namespace eckit::geo::iterator { class Regular : public Iterator { public: // -- Constructors explicit Regular(const grid::Regular&, size_t index = 0); private: // -- Members const grid::Regular& grid_; const Projection& projection_; const std::vector& x_; const std::vector& y_; size_t ix_; size_t iy_; size_t index_; const size_t nx_; const size_t ny_; const size_t size_; // -- Overridden methods bool operator==(const Iterator&) const override; bool operator++() override; bool operator+=(difference_type) override; explicit operator bool() const override; Point operator*() const override; size_t index() const override { return index_; } }; } // namespace eckit::geo::iterator eckit-2.0.7/src/eckit/geo/iterator/Regular.cc0000664000175000017500000000365515161702250021214 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/iterator/Regular.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/grid/Regular.h" namespace eckit::geo::iterator { struct Instance { explicit Instance(const Grid::Spec& spec) : grid(dynamic_cast(GridFactory::build(spec))) { ASSERT(grid); } std::unique_ptr grid; }; struct RegularInstance : Instance, Regular { explicit RegularInstance(const Grid::Spec& spec) : Instance(spec), Regular(*grid) {} }; Regular::Regular(const grid::Regular& grid, size_t index) : grid_(grid), projection_(grid_.projection()), x_(grid.x().values()), y_(grid.y().values()), ix_(0), iy_(0), index_(index), nx_(x_.size()), ny_(y_.size()), size_(nx_ * ny_) {} bool Regular::operator==(const Iterator& other) const { const auto* another = dynamic_cast(&other); return another != nullptr && index_ == another->index_; } bool Regular::operator++() { if (index_++, ix_++; index_ < size_) { if (ix_ >= nx_) { ix_ = 0; iy_++; } return true; } index_ = size_; // ensure it's invalid return false; } bool Regular::operator+=(difference_type d) { NOTIMP; } Regular::operator bool() const { return index_ < size_; } Point Regular::operator*() const { return projection_.fwd(PointXY{x_.at(ix_), y_.at(iy_)}); } static const IteratorRegisterType ITERATOR_TYPE("regular"); } // namespace eckit::geo::iterator eckit-2.0.7/src/eckit/geo/Iterator.h0000664000175000017500000000534415161702250017412 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/Point.h" #include "eckit/memory/Builder.h" #include "eckit/memory/Factory.h" #include "eckit/spec/Custom.h" namespace eckit::geo { class Grid; } namespace eckit::geo { class Iterator { public: // -- Types using builder_t = BuilderT1; using Spec = spec::Spec; using ARG1 = const Spec&; using iterator_category = std::input_iterator_tag; using difference_type = std::ptrdiff_t; using value_type = Point; using pointer = value_type*; using reference = value_type&; // -- Constructors Iterator(const Iterator&) = default; Iterator(Iterator&&) = default; // -- Destructor virtual ~Iterator() = default; // -- Operators Iterator& operator=(const Iterator&) = default; Iterator& operator=(Iterator&&) = default; virtual bool operator==(const Iterator&) const = 0; bool operator!=(const Iterator& other) const { return !operator==(other); } virtual bool operator++() = 0; virtual bool operator+=(difference_type) = 0; virtual bool operator--() { return operator-=(1); } virtual bool operator-=(difference_type diff) { return operator+=(-diff); } virtual explicit operator bool() const = 0; virtual value_type operator*() const = 0; // -- Methods virtual size_t index() const = 0; // -- Class methods static std::string className() { return "iterator"; } protected: // -- Constructors Iterator() = default; private: // -- Friends friend class Grid; }; using IteratorFactoryType = Factory; template using IteratorRegisterType = ConcreteBuilderT1; struct IteratorFactory { [[nodiscard]] static Iterator* build(const Iterator::Spec& spec) { return instance().build_(spec); } [[nodiscard]] static Iterator::Spec* make_spec(const Iterator::Spec& spec) { return instance().make_spec_(spec); } static std::ostream& list(std::ostream& out) { return instance().list_(out); } private: static IteratorFactory& instance(); [[nodiscard]] Iterator* build_(const Iterator::Spec&) const; [[nodiscard]] Iterator::Spec* make_spec_(const Iterator::Spec&) const; std::ostream& list_(std::ostream&) const; }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Trace.h0000664000175000017500000000265015161702250016654 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/log/Seconds.h" #include "eckit/log/Timer.h" namespace eckit { class ResourceUsage; } namespace eckit::geo { class Trace : public Timer { public: explicit Trace(const std::string&); Trace(const Trace&) = delete; Trace(Trace&&) = delete; virtual ~Trace() = default; void operator=(const Trace&) = delete; void operator=(Trace&&) = delete; using Timer::elapsed; double elapsed(double t) { return Timer::elapsed() - t; } Seconds elapsedSeconds(double t = 0, bool compact = false) { return {elapsed(t), compact}; } }; struct TraceResourceUsage final : public Trace { explicit TraceResourceUsage(const std::string&); TraceResourceUsage(const TraceResourceUsage&) = delete; TraceResourceUsage(TraceResourceUsage&&) = delete; ~TraceResourceUsage(); void operator=(const TraceResourceUsage&) = delete; void operator=(TraceResourceUsage&&) = delete; private: ResourceUsage* info_; }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Grid.cc0000664000175000017500000002075415161702250016646 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Grid.h" #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/projection/EquidistantCylindrical.h" #include "eckit/geo/projection/Reverse.h" #include "eckit/geo/share/Grid.h" #include "eckit/geo/util/mutex.h" #include "eckit/log/Log.h" #include "eckit/parser/YAMLParser.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Layered.h" #include "eckit/utils/MD5.h" namespace eckit::geo { namespace { // 128-bit hash = 32 hex characters * 4 bits per character constexpr size_t DIGEST_LENGTH = 32; static_assert(DIGEST_LENGTH == MD5_DIGEST_LENGTH * 2, "MD5 digest length mismatch"); util::recursive_mutex MUTEX; class lock_type { util::lock_guard lock_guard_{MUTEX}; }; } // namespace Grid::Grid(Projection* proj) : projection_(proj == nullptr ? new projection::Reverse : proj) {} const spec::Spec& Grid::catalog() const { if (!catalog_) { if (GridSpecByUID::instance().exists(uid())) { catalog_.reset(GridSpecByUID::instance().get(uid()).spec()); } else if (std::string grid(spec().get_string("grid")); GridSpecByName::instance().matches(grid)) { catalog_.reset(GridSpecByName::instance().match(grid).spec(grid)); } else { static const spec::Custom empty; catalog_.reset(&empty); } } ASSERT(catalog_); return *catalog_; } const Grid::Spec& Grid::spec() const { if (!spec_) { spec_ = std::make_unique(); ASSERT(spec_); auto& custom = *spec_; fill_spec(custom); } return *spec_; } bool Grid::empty() const { return size() == 0; } size_t Grid::size() const { NOTIMP; } bool Grid::is_uid(const std::string& str) { return str.length() == DIGEST_LENGTH && std::all_of(str.begin(), str.end(), [](char c) { return std::isxdigit(static_cast(c)); }); } Grid::uid_type Grid::uid() const { if (uid_.empty()) { const_cast(this)->reset_uid(calculate_uid()); } return uid_; } Grid::uid_type Grid::calculate_uid() const { auto id = MD5{spec_str()}.digest(); std::transform(id.begin(), id.end(), id.begin(), [](unsigned char c) { return std::tolower(c); }); ASSERT(is_uid(id)); return id; } void Grid::reset_uid(uid_type id) { if (id.empty()) { uid_.clear(); return; } ASSERT(is_uid(id)); std::transform(id.begin(), id.end(), id.begin(), [](unsigned char c) { return std::tolower(c); }); uid_ = id; } bool Grid::includesNorthPole() const { NOTIMP; } bool Grid::includesSouthPole() const { NOTIMP; } bool Grid::isPeriodicWestEast() const { NOTIMP; } Point Grid::first_point() const { ASSERT(!empty()); return to_points().front(); } Point Grid::last_point() const { ASSERT(!empty()); return to_points().back(); } std::vector Grid::to_points() const { return {cbegin(), cend()}; } std::pair, std::vector> Grid::to_latlons() const { std::pair, std::vector> ll; ll.first.reserve(size()); ll.second.reserve(size()); std::for_each(cbegin(), cend(), [&ll](const auto& p) { auto q = std::get(p); ll.first.emplace_back(q.lat()); ll.second.emplace_back(q.lon()); }); return ll; } const Grid::order_type& Grid::order() const { NOTIMP; } Grid::renumber_type Grid::reorder(const order_type&) const { NOTIMP; } Grid* Grid::make_grid_reordered(const order_type&) const { NOTIMP; } const Area& Grid::area() const { if (!bbox_) { bbox_.reset(calculate_bbox()); ASSERT(bbox_); } return *bbox_; } Grid::renumber_type Grid::crop(const Area&) const { NOTIMP; } const Projection& Grid::projection() const { if (!projection_) { projection_ = std::make_unique>(); ASSERT(projection_); } return *projection_; } Grid* Grid::make_grid_cropped(const Area&) const { NOTIMP; } const Grid::BoundingBox& Grid::boundingBox() const { if (!bbox_) { bbox_.reset(calculate_bbox()); ASSERT(bbox_); } return *bbox_; } Grid::BoundingBox* Grid::calculate_bbox() const { NOTIMP; } void Grid::fill_spec(spec::Custom& custom) const { auto custom_set_if_different = [&custom](const std::string& name, const auto& obj, const std::string& default_str) { spec::Custom spec; obj.fill_spec(spec); if (default_str != spec.str()) { if (spec.only(name)) { custom.set(spec); } else { custom.set(name, spec); } } }; static const auto area_default = Area::area_default().spec().str(); static const auto proj_default = Projection::projection_default().spec().str(); custom_set_if_different("area", area(), area_default); custom_set_if_different("projection", projection(), proj_default); } const Grid* GridFactory::make_from_string(const std::string& str) { std::unique_ptr spec(spec::Custom::make_from_value(YAMLParser::decodeString(str))); return instance().make_from_spec_(*spec); } GridFactory& GridFactory::instance() { share::Grid::instance(); // ensure load of supporting files static GridFactory INSTANCE; return INSTANCE; } const Grid* GridFactory::make_from_spec_(const Grid::Spec& spec) const { lock_type lock; std::unique_ptr cfg(make_spec_(spec)); if (std::string type; cfg->get("type", type)) { return Factory::instance().get(type).create(*cfg); } list(Log::error() << "Grid: cannot build grid without 'type', choices are: "); throw exception::SpecError("Grid: cannot build grid without 'type'", Here()); } Grid::Spec* GridFactory::make_spec_(const Grid::Spec& spec) const { lock_type lock; auto* cfg = new spec::Layered(spec); ASSERT(cfg != nullptr); // hardcoded, interpreted options (contributing to spec) auto back = std::make_unique(); ASSERT(back); if (size_t N = 0; cfg->get("N", N)) { back->set("grid", "O" + std::to_string(N)); } if (cfg->has("pl")) { back->set("type", "reduced_gg"); } if (std::vector grid; cfg->get("grid", grid) && grid.size() == 2) { back->set("type", "regular_ll"); } if (static const std::string projection{"projection"}; !cfg->has(projection)) { auto ptr = std::make_unique(); ASSERT(ptr); if (static const std::string rotation{"rotation"}; cfg->has(rotation)) { ptr->set("type", rotation); ptr->set(rotation, cfg->get_double_vector(rotation)); } else { ptr->set("type", "none"); } back->set(projection, ptr.release()); } if (!back->empty()) { cfg->push_back(back.release()); } if (std::string grid; cfg->get("grid", grid) && GridSpecByName::instance().matches(grid)) { cfg->push_back(GridSpecByName::instance().match(grid).spec(grid)); } if (std::string uid; cfg->get("uid", uid) || (cfg->get("grid", uid) && Grid::is_uid(uid))) { if (!GridSpecByUID::instance().exists(uid)) { throw exception::GridUnknownError("Grid: unknown grid uid '" + uid + "'", Here()); } cfg->push_front(GridSpecByUID::instance().get(uid).spec()); } return cfg; } std::ostream& GridFactory::list_(std::ostream& out) const { lock_type lock; out << GridSpecByUID::instance() << std::endl; out << GridSpecByName::instance() << std::endl; out << Factory::instance() << std::endl; return out; } GridSpecByName::generator_t& GridSpecByName::instance() { share::Grid::instance(); // ensure load of supporting files return generator_t::instance(); } GridSpecByUID::generator_t& GridSpecByUID::instance() { share::Grid::instance(); // ensure load of supporting files return generator_t::instance(); } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/PointLonLatR.cc0000664000175000017500000000515415161702250020303 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/PointLonLatR.h" #include "eckit/geo/util.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo { PointLonLatR::value_type PointLonLatR::normalise_angle_to_minimum(value_type a, value_type minimum) { static const auto modulus = [](auto a) { return a - FULL_ANGLE * std::floor(a / FULL_ANGLE); }; auto diff = a - minimum; return 0. <= diff && diff < FULL_ANGLE ? a : (modulus(diff) + minimum); } PointLonLatR::value_type PointLonLatR::normalise_angle_to_maximum(value_type a, value_type maximum) { auto modulus = [](auto a) { return a - FULL_ANGLE * std::ceil(a / FULL_ANGLE); }; auto diff = a - maximum; return -FULL_ANGLE < diff && diff <= 0. ? a : (modulus(diff) + maximum); } bool PointLonLatR::pole(value_type eps) const { const auto p = make(lonr(), latr()); return types::is_approximately_equal(p.latr(), RIGHT_ANGLE, eps) || types::is_approximately_equal(p.latr(), -RIGHT_ANGLE, eps); } PointLonLatR PointLonLatR::make(value_type lonr, value_type latr, value_type lonr_minimum, value_type eps) { latr = normalise_angle_to_minimum(latr, -RIGHT_ANGLE); if (types::is_strictly_greater(latr, RIGHT_ANGLE, eps)) { latr = FLAT_ANGLE - latr; lonr += FLAT_ANGLE; } return types::is_approximately_equal(latr, RIGHT_ANGLE, eps) ? NORTH_POLE_R : types::is_approximately_equal(latr, -RIGHT_ANGLE, eps) ? SOUTH_POLE_R : PointLonLatR{normalise_angle_to_minimum(lonr, lonr_minimum), latr}; } PointLonLatR PointLonLatR::make_from_lonlat(value_type lon, value_type lat, value_type lonr_minimum) { return make(util::DEGREE_TO_RADIAN * lon, util::DEGREE_TO_RADIAN * lat, lonr_minimum); } bool points_equal(const PointLonLatR& a, const PointLonLatR& b, PointLonLatR::value_type eps) { const auto c = PointLonLatR::make(a.lonr(), a.latr(), 0., eps); const auto d = PointLonLatR::make(b.lonr(), b.latr(), 0., eps); return types::is_approximately_equal(c.lonr(), d.lonr(), eps) && types::is_approximately_equal(c.latr(), d.latr(), eps); } const PointLonLatR NORTH_POLE_R{0., PointLonLatR::RIGHT_ANGLE}; const PointLonLatR SOUTH_POLE_R{0., -PointLonLatR::RIGHT_ANGLE}; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Area.cc0000664000175000017500000000701315161702250016622 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Area.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/area/BoundingBox.h" #include "eckit/geo/share/Area.h" #include "eckit/geo/util/mutex.h" #include "eckit/parser/YAMLParser.h" #include "eckit/spec/Layered.h" namespace eckit::geo { namespace { util::recursive_mutex MUTEX; class lock_type { util::lock_guard lock_guard_{MUTEX}; }; } // namespace const Area::Spec& Area::spec() const { if (!spec_) { spec_ = std::make_shared(); ASSERT(spec_); auto& custom = *spec_; fill_spec(custom); if (std::string name; !custom.empty() && AreaSpecByName::instance().match(custom, name)) { custom.clear(); custom.set(className(), name); } } return *spec_; } bool Area::intersects(area::BoundingBox&) const { NOTIMP; } bool Area::contains(const Point&) const { NOTIMP; } double Area::area() const { NOTIMP; } const Area& Area::area_default() { return area::BoundingBox::bounding_box_default(); } const Area* AreaFactory::make_from_string(const std::string& str) { std::unique_ptr spec(spec::Custom::make_from_value(YAMLParser::decodeString(str))); return instance().make_from_spec_(*spec); } AreaFactory& AreaFactory::instance() { static AreaFactory obj; return obj; } const Area* AreaFactory::make_from_spec_(const spec::Spec& spec) const { lock_type lock; std::unique_ptr cfg(make_spec_(spec)); if (std::string type; cfg->get("type", type)) { return AreaFactoryType::instance().get(type).create(*cfg); } list(Log::error() << "Area: cannot build area without 'type', choices are: "); throw exception::SpecError("Area: cannot build area without 'type'", Here()); } Area::Spec* AreaFactory::make_spec_(const Area::Spec& spec) const { lock_type lock; share::Area::instance(); auto* cfg = new spec::Layered(spec); ASSERT(cfg != nullptr); // hardcoded, interpreted options (contributing to areaspec) auto back = std::make_unique(); ASSERT(back); cfg->push_back(new spec::Custom{{"type", "bounding_box"}}); // if (cfg->has("north") || cfg->has("east") || cfg->has("south") || cfg->has("west")) { // // back->set("type", "reduced_gg"); // } // if (std::vector area; cfg->get("area", area) && area.size() == 4) { // back->set("type", "regular_ll"); // } if (!back->empty()) { cfg->push_back(back.release()); } return cfg; } void AreaFactory::add_library_(const std::string& lib, Area::Spec* spec) { lock_type lock; share::Area::instance(); libraries_.emplace(lib, spec); } std::ostream& AreaFactory::list_(std::ostream& out) const { lock_type lock; share::Area::instance(); out << AreaSpecByName::instance() << std::endl; out << AreaFactoryType::instance() << std::endl; out << "Libraries:" << std::endl; for (const auto& [name, spec] : libraries_) { out << " " << name << ": " << *spec << std::endl; } return out; } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/polygon/0000775000175000017500000000000015161702250017131 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/polygon/Polygon.h0000664000175000017500000000555615161702250020744 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/geo/PointLonLat.h" namespace eckit::geo::area { class BoundingBox; } namespace eckit::geo::polygon { class Polygon : public std::vector { public: // -- Types using container_type = vector; using container_type::value_type; // -- Constructors using container_type::container_type; explicit Polygon(const container_type& points) : container_type(points) {} explicit Polygon(container_type&& points) : container_type(std::move(points)) {} Polygon(const Polygon&) = default; Polygon(Polygon&&) = default; // -- Destructor ~Polygon() = default; // -- Operators Polygon& operator=(const Polygon&) = default; Polygon& operator=(Polygon&&) = default; bool operator==(const Polygon&) const; bool operator!=(const Polygon& other) const { return !(*this == other); } // -- Methods using container_type::operator[]; using container_type::size; /** * @brief Intersect polygon with bounding box * @param[in,out] bbox bounding box to intersect with, returns intersection * @return if polygon intersects bounding box */ bool intersects(area::BoundingBox& bbox) const; /** * @brief Point-in-polygon test based on winding number * @param[in] P given point * @return if point is in polygon */ bool contains(const PointLonLat& P) const; /** * @brief Centroid of polygon * @return centroid */ PointLonLat centroid() const; /** * @brief Clip polygon with another polygon (commutative) * @param[in] other polygon to clip with */ void clip(const Polygon&); /** * @brief Simplify polygon by removing consecutive and colinear points */ void simplify(); /** * @brief Area of polygon * @param[in] sign if true, returns positive area irrespective of point ordering * @return area respecting point ordering (positive on CCW point ordering) */ double area(bool sign = false) const; private: // -- Types using Edge = std::pair; // -- Methods Edge edge(int) const; void emplace_back_point(PointLonLat); void emplace_back_point_at_intersection(const Edge&, const Edge&); void print(std::ostream&) const; // -- Friends friend std::ostream& operator<<(std::ostream&, const Polygon&); }; } // namespace eckit::geo::polygon eckit-2.0.7/src/eckit/geo/polygon/Polygon.cc0000664000175000017500000001560615161702250021077 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/polygon/Polygon.h" #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/PointXY.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::polygon { namespace { inline bool is_zero(double val) { return types::is_approximately_equal(val, 0., PointLonLat::EPS); } inline double cross_product_analog(const PointLonLat& A, const PointLonLat& B, const PointLonLat& C) { return (A.lon() - C.lon()) * (B.lat() - C.lat()) - (A.lat() - C.lat()) * (B.lon() - C.lon()); } inline double cross(const PointLonLat& P, const PointLonLat& Q) { return P.lon() * Q.lat() - P.lat() * Q.lon(); } inline int on_direction(double a, double b, double c) { return a <= b && b <= c ? 1 : c <= b && b <= a ? -1 : 0; }; inline int on_side(const PointLonLat& P, const PointLonLat& A, const PointLonLat& B) { const auto p = cross_product_analog(P, A, B); return is_zero(p) ? 0 : p > 0 ? 1 : -1; } } // namespace bool Polygon::operator==(const Polygon& other) const { if (size() == other.size()) { if (empty()) { return true; } auto i = std::distance(begin(), std::min_element(begin(), end())); auto j = std::distance(other.begin(), std::min_element(other.begin(), other.end())); for (int k = 0, n = static_cast(size()); k < n; ++k) { if (!points_equal(operator[]((i + k) % n), other[(j + k) % n])) { return false; } } return true; } return false; } bool Polygon::intersects(area::BoundingBox& bbox) const { NOTIMP; } Polygon::Edge Polygon::edge(int i) const { const auto n = static_cast(size()); return {at(((i % n) + n) % n), at(((i % n) + n + 1) % n)}; } void Polygon::emplace_back_point(PointLonLat P) { if (empty() || (!points_equal(P, back()) && !points_equal(P, front()))) { emplace_back(P); } } void Polygon::emplace_back_point_at_intersection(const Edge& E, const Edge& F) { const auto A = E.second - E.first; const auto B = F.second - F.first; if (const auto D = cross(A, B); !is_zero(D)) { const auto C = E.first - F.first; emplace_back_point(E.first + A * cross(B, C) * (1. / D)); } } void Polygon::print(std::ostream& out) const { out << "["; const auto* sep = ""; for (const auto& p : *this) { out << sep << p; sep = ", "; } out << "]"; } std::ostream& operator<<(std::ostream& out, const Polygon& pc) { pc.print(out); return out; } bool Polygon::contains(const PointLonLat& P) const { // Winding number algorithm for point-in-polygon test PointLonLat::assert_latitude_range(P); if (empty()) { return false; } auto min_lon = front().lon(); auto max_lon = min_lon; std::for_each(begin(), end(), [&](const auto& P) { min_lon = std::min(P.lon(), min_lon); max_lon = std::max(P.lon(), max_lon); }); auto Q = PointLonLat::make(P.lon(), P.lat(), min_lon); do { int wn = 0; int prev = 0; // loop on polygon edges, check point-edge side and direction, testing if P is on|above|below (in latitude) of a // A,B polygon edge, by: // - intersecting "up" on forward crossing & P above edge, or // - intersecting "down" on backward crossing & P below edge for (int i = 0, n = static_cast(size()); i < n; ++i) { const auto& [A, B] = edge(i); if (const auto dir = on_direction(A.lat(), Q.lat(), B.lat()); dir != 0) { const auto side = on_side(Q, A, B); if (side == 0 && on_direction(A.lon(), Q.lon(), B.lon()) != 0) { return true; } if ((prev != 1 && dir > 0 && side > 0) || (prev != -1 && dir < 0 && side < 0)) { prev = dir; wn += dir; } } } // wn == 0 only when P is outside if (wn != 0) { return true; } Q = {Q.lon() + 360., Q.lat()}; } while (Q.lon() <= max_lon); return false; } PointLonLat Polygon::centroid() const { double a = 0.; PointLonLat C{0., 0.}; for (int i = 0, n = static_cast(size()); i < n; ++i) { auto ei = edge(i); auto ai = cross(ei.first, ei.second); auto Ci = (ei.first + ei.second) * ai; C = {C.lon() + Ci.lon(), C.lat() + Ci.lat()}; a += ai; } return is_zero(a) ? C : C * (1. / (3. * a)); } void Polygon::clip(const Polygon& clipper) { // Sutherland-Hodgman algorithm for clipping polygons if (empty() || clipper.empty()) { clear(); return; } auto is_point_left_of_edge = [](const Edge& E, const PointLonLat& P) { const auto r = cross(E.second - E.first, P - E.first); return 0. <= r || is_zero(r); }; for (int i = 0, m = static_cast(clipper.size()); i < m; ++i) { const auto c = clipper.edge(i); Polygon poly; swap(poly); for (int j = 0, n = static_cast(poly.size()); j < n; ++j) { const auto p = poly.edge(j); if (is_point_left_of_edge(c, p.second)) { if (!is_point_left_of_edge(c, p.first)) { emplace_back_point_at_intersection(c, p); } emplace_back_point(p.second); } else if (is_point_left_of_edge(c, p.first)) { emplace_back_point_at_intersection(c, p); } } } simplify(); } void Polygon::simplify() { // remove consecutive duplicate points erase(std::unique(begin(), end(), [](const auto& P, const auto& Q) { return points_equal(P, Q); }), end()); if (1 < size() && points_equal(front(), back())) { pop_back(); } // remove consecutive colinear points Polygon poly; swap(poly); reserve(poly.size()); for (int i = 0, n = static_cast(poly.size()); i < n; ++i) { if (const auto E = poly.edge(i), F = poly.edge(i + 1); !is_zero(cross(E.second - E.first, F.second - E.second))) { emplace_back_point(E.second); } } if (size() < 3) { clear(); } } double Polygon::area(bool sign) const { double a = 0; if (3 <= size()) { for (int i = 0, n = static_cast(size()); i < n; ++i) { const auto E = edge(i); a += cross(E.first, E.second); } } return (sign ? a : std::abs(a)) / 2.; } } // namespace eckit::geo::polygon eckit-2.0.7/src/eckit/geo/polygon/PolygonXY.cc0000664000175000017500000001001015161702250021340 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/polygon/PolygonXY.h" #include #include #include "eckit/geo/Exceptions.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::polygon { namespace { inline bool is_zero(double val) { return types::is_approximately_equal(val, 0., PointXY::EPS); } inline double cross_product_analog(const PointXY& A, const PointXY& B, const PointXY& C) { return (A.X() - C.X()) * (B.Y() - C.Y()) - (A.Y() - C.Y()) * (B.X() - C.X()); } inline double cross(const PointXY& P, const PointXY& Q) { return P.X() * Q.Y() - P.Y() * Q.X(); } inline int on_direction(double a, double b, double c) { return a <= b && b <= c ? 1 : c <= b && b <= a ? -1 : 0; }; inline int on_side(const PointXY& P, const PointXY& A, const PointXY& B) { const auto p = cross_product_analog(P, A, B); return is_zero(p) ? 0 : p > 0 ? 1 : -1; } } // namespace bool PolygonXY::operator==(const PolygonXY& poly) const { // congruence test if (size() == poly.size()) { if (empty()) { return true; } int offset = -1; for (int i = 0; i < size(); i++) { if (at(i) == poly.at(0)) { offset = i; break; } } if (offset == -1) { return false; } for (int i = 1; i < size(); i++) { if (at((i + offset) % size()) != poly.at(i)) { return false; } } return true; } return false; } bool PolygonXY::contains(const PointXY& P) const { // Winding number algorithm for point-in-polygon test // loop on polygon edges, check point-edge side and direction, testing if P is on|above|below of a A,B polygon edge, // by: // - intersecting "up" on forward crossing & P above edge, or // - intersecting "down" on backward crossing & P below edge int wn = 0; for (int i = 0, n = static_cast(size()), prev = 0; i < n; ++i) { const auto& [A, B] = edge(i); if (const auto dir = on_direction(A.Y(), P.Y(), B.Y()); dir != 0) { const auto side = on_side(P, A, B); if (side == 0 && on_direction(A.X(), P.X(), B.X()) != 0) { return true; } if ((prev != 1 && dir > 0 && side > 0) || (prev != -1 && dir < 0 && side < 0)) { prev = dir; wn += dir; } } } return wn != 0; } void PolygonXY::simplify() { // remove consecutive duplicate points erase(std::unique(begin(), end(), [](const auto& P, const auto& Q) { return points_equal(P, Q); }), end()); if (1 < size() && points_equal(front(), back())) { pop_back(); } // remove consecutive colinear points PolygonXY poly; swap(poly); for (int i = 0, n = static_cast(poly.size()); i < n; ++i) { if (const auto E = poly.edge(i), F = poly.edge(i + 1); !is_zero(cross(E.second - E.first, F.second - E.second))) { emplace_back_point(E.second); } } if (size() < 3) { clear(); } } void PolygonXY::emplace_back_point(PointXY P) { if (empty() || (!points_equal(P, back()) && !points_equal(P, front()))) { emplace_back(P); } } PolygonXY::Edge PolygonXY::edge(int i) const { const auto n = static_cast(size()); return {at(((i % n) + n) % n), at(((i % n) + n + 1) % n)}; } void PolygonXY::print(std::ostream& s) const { if (empty()) { s << "[]"; return; } char z = '['; for (const auto& v : *this) { s << z << v; z = ','; } s << ']'; } } // namespace eckit::geo::polygon eckit-2.0.7/src/eckit/geo/polygon/PolygonXY.h0000664000175000017500000000374515161702250021223 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/geo/PointXY.h" namespace eckit::geo::polygon { class PolygonXY : public std::vector { public: // -- Types using container_type = vector; using container_type::value_type; // -- Constructors using container_type::container_type; explicit PolygonXY(const container_type& points) : container_type(points) {} explicit PolygonXY(container_type&& points) : container_type(std::move(points)) {} PolygonXY(const PolygonXY&) = default; PolygonXY(PolygonXY&&) = default; // -- Destructor ~PolygonXY() = default; // -- Operators PolygonXY& operator=(const PolygonXY&) = default; PolygonXY& operator=(PolygonXY&&) = default; bool operator==(const PolygonXY&) const; bool operator!=(const PolygonXY& other) const { return !(*this == other); } // -- Methods /** * @brief Point-in-polygon test based on winding number * @param[in] P given point * @return if point is in polygon */ bool contains(const PointXY& P) const; /** * @brief Simplify polygon by removing consecutive and colinear points */ void simplify(); private: // -- Types using Edge = std::pair; // -- Methods Edge edge(int) const; void emplace_back_point(PointXY); void print(std::ostream&) const; // -- Friends friend std::ostream& operator<<(std::ostream& s, const PolygonXY& p) { p.print(s); return s; } }; } // namespace eckit::geo::polygon eckit-2.0.7/src/eckit/geo/Shape.h0000664000175000017500000000350715161702250016660 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit::spec { class Spec; } namespace eckit::geo { class Shape final : public std::array { public: // -- Types using container_type = array; // -- Constructors explicit Shape(const spec::Spec& spec) : Shape(make_from_spec(spec)) {} Shape(value_type nx, value_type ny); Shape() : Shape(0, 0) {} Shape(const Shape& other) : container_type(other) {} Shape(Shape&& other) : container_type(other) {} // -- Destructor ~Shape() = default; // -- Operators bool operator==(const Shape& other) const { return nx == other.nx && ny == other.ny; } bool operator!=(const Shape& other) const { return !operator==(other); } Shape& operator=(const Shape& other) { container_type::operator=(other); return *this; } Shape& operator=(Shape&& other) { container_type::operator=(other); return *this; } // Members const value_type& nx = container_type::operator[](0); const value_type& ny = container_type::operator[](1); // -- Methods container_type deconstruct() const { return {nx, ny}; } // -- Class methods static Shape make_from_spec(const spec::Spec&); private: // -- Friends friend std::ostream& operator<<(std::ostream& os, const Shape& inc) { return os << "[" << inc.nx << "," << inc.ny << "]"; } }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Iterator.cc0000664000175000017500000000350315161702250017543 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Iterator.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/util/mutex.h" #include "eckit/spec/Layered.h" namespace eckit::geo { namespace { util::recursive_mutex MUTEX; class lock_type { util::lock_guard lock_guard_{MUTEX}; }; } // namespace IteratorFactory& IteratorFactory::instance() { static IteratorFactory INSTANCE; return INSTANCE; } Iterator* IteratorFactory::build_(const Iterator::Spec& spec) const { lock_type lock; std::unique_ptr cfg(make_spec_(spec)); if (std::string type; cfg->get("type", type)) { return IteratorFactoryType::instance().get(type).create(*cfg); } list(Log::error() << "Iterator: cannot build iterator without 'type', choices are: "); throw exception::SpecError("Iterator: cannot build iterator without 'type'", Here()); } Iterator::Spec* IteratorFactory::make_spec_(const Iterator::Spec& spec) const { lock_type lock; auto* cfg = new spec::Layered(spec); ASSERT(cfg != nullptr); // hardcoded, interpreted options (contributing to spec) auto back = std::make_unique(); ASSERT(back); back->set("type", cfg->has("pl") ? "reduced" : "regular"); return cfg; } std::ostream& IteratorFactory::list_(std::ostream& out) const { lock_type lock; out << IteratorFactoryType::instance() << std::endl; return out; } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/share/0000775000175000017500000000000015161702250016544 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/share/Projection.h0000664000175000017500000000156515161702250021040 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit { namespace spec { class Spec; } class PathName; } // namespace eckit namespace eckit::geo::share { class Projection final { public: static const Projection& instance(); private: // -- Constructors explicit Projection(const std::vector&); // -- Members std::unique_ptr spec_; // -- Methods void load(const PathName&); }; } // namespace eckit::geo::share eckit-2.0.7/src/eckit/geo/share/Grid.cc0000664000175000017500000000627115161702250017746 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/share/Grid.h" #include "eckit/filesystem/PathName.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/Grid.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/log/Log.h" #include "eckit/parser/YAMLParser.h" #include "eckit/spec/Custom.h" #include "eckit/value/Value.h" namespace eckit::geo::share { const Grid& Grid::instance() { static const Grid INSTANCE(LibEcKitGeo::shareGrid()); return INSTANCE; } Grid::Grid(const std::vector& paths) : spec_(new spec::Custom) { ASSERT(spec_); for (const auto& path : paths) { if (path.exists()) { Log::debug() << "eckit::geo::share::Grid::load('" << path.realName() << "')" << std::endl; load(path); } } } void Grid::load(const PathName& path) { auto* custom = dynamic_cast(spec_.get()); ASSERT(custom != nullptr); struct SpecByUIDGenerator final : GridSpecByUID::concrete_generator_t { explicit SpecByUIDGenerator(spec::Custom* spec) : spec_(spec) { ASSERT(spec_); } spec::Spec* spec() const override { return new spec::Custom(spec_->container()); } bool match(const spec::Custom& other) const override { return other == *spec_; } private: std::unique_ptr spec_; }; struct SpecByNameGenerator final : GridSpecByName::concrete_generator_t { explicit SpecByNameGenerator(spec::Custom* spec) : spec_(spec) { ASSERT(spec_); } spec::Spec* spec(GridSpecByName::concrete_generator_t::arg1_t) const override { return new spec::Custom(spec_->container()); } bool match(const spec::Custom& other) const override { return other == *spec_; } private: std::unique_ptr spec_; }; if (path.exists()) { ValueMap map(YAMLParser::decodeFile(path)); for (const auto& kv : map) { const auto key = kv.first.as(); if (key == "grid_uids") { for (ValueMap m : kv.second.as()) { ASSERT(m.size() == 1); GridSpecByUID::regist(m.begin()->first.as(), new SpecByUIDGenerator(spec::Custom::make_from_value(m.begin()->second))); } continue; } if (key == "grid_names") { for (ValueMap m : kv.second.as()) { ASSERT(m.size() == 1); GridSpecByName::regist(m.begin()->first.as(), new SpecByNameGenerator(spec::Custom::make_from_value(m.begin()->second))); } continue; } custom->set(key, kv.second); } } } } // namespace eckit::geo::share eckit-2.0.7/src/eckit/geo/share/Area.cc0000664000175000017500000000527715161702250017736 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/share/Area.h" #include "eckit/filesystem/PathName.h" #include "eckit/geo/Area.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/log/Log.h" #include "eckit/parser/YAMLParser.h" #include "eckit/spec/Custom.h" #include "eckit/value/Value.h" namespace eckit::geo::share { const Area& Area::instance() { static const Area INSTANCE(LibEcKitGeo::shareArea()); return INSTANCE; } Area::Area(const std::vector& paths) : spec_(new spec::Custom) { ASSERT(spec_); for (const auto& path : paths) { if (path.exists()) { Log::debug() << "eckit::geo::share::Area::load('" << path.realName() << "')" << std::endl; load(path); } } } void Area::load(const PathName& path) { auto* custom = dynamic_cast(spec_.get()); ASSERT(custom != nullptr); struct SpecByNameGenerator final : AreaSpecByName::generator_t { explicit SpecByNameGenerator(spec::Custom* spec) : spec_(spec) { ASSERT(spec_); } geo::Area::Spec* spec(AreaSpecByName::generator_t::arg1_t) const override { return new spec::Custom(spec_->container()); } bool match(const spec::Custom& other) const override { return other == *spec_; } private: std::unique_ptr spec_; }; if (path.exists()) { ValueMap map(YAMLParser::decodeFile(path)); for (const auto& kv : map) { const auto key = kv.first.as(); if (key == "area_libraries") { for (const auto& [name, spec] : kv.second.as()) { AreaFactory::add_library(name.as(), spec::Custom::make_from_value(spec)); } continue; } if (key == "area_names") { for (ValueMap named : kv.second.as()) { ASSERT(named.size() == 1); AreaSpecByName::instance().regist( named.begin()->first.as(), new SpecByNameGenerator(spec::Custom::make_from_value(named.begin()->second))); } continue; } custom->set(key, kv.second); } } } } // namespace eckit::geo::share eckit-2.0.7/src/eckit/geo/share/Grid.h0000664000175000017500000000154315161702250017605 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit { namespace spec { class Spec; } class PathName; } // namespace eckit namespace eckit::geo::share { class Grid final { public: static const Grid& instance(); private: // -- Constructors explicit Grid(const std::vector&); // -- Members std::unique_ptr spec_; // -- Methods void load(const PathName&); }; } // namespace eckit::geo::share eckit-2.0.7/src/eckit/geo/share/Projection.cc0000664000175000017500000000502215161702250021166 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/share/Projection.h" #include "eckit/filesystem/PathName.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/geo/Projection.h" #include "eckit/log/Log.h" #include "eckit/parser/YAMLParser.h" #include "eckit/spec/Custom.h" #include "eckit/value/Value.h" namespace eckit::geo::share { const Projection& Projection::instance() { static const Projection INSTANCE(LibEcKitGeo::shareProjection()); return INSTANCE; } Projection::Projection(const std::vector& paths) : spec_(new spec::Custom) { ASSERT(spec_); for (const auto& path : paths) { if (path.exists()) { Log::debug() << "eckit::geo::share::Projection::load('" << path.realName() << "')" << std::endl; load(path); } } } void Projection::load(const PathName& path) { auto* custom = dynamic_cast(spec_.get()); ASSERT(custom != nullptr); struct SpecByNameGenerator final : ProjectionSpecByName::generator_t { explicit SpecByNameGenerator(spec::Custom* spec) : spec_(spec) { ASSERT(spec_); } geo::Projection::Spec* spec(ProjectionSpecByName::generator_t::arg1_t) const override { return new spec::Custom(spec_->container()); } bool match(const spec::Custom& other) const override { return other == *spec_; } private: std::unique_ptr spec_; }; if (path.exists()) { ValueMap map(YAMLParser::decodeFile(path)); for (const auto& kv : map) { const auto key = kv.first.as(); if (key == "projection_names") { for (ValueMap m : kv.second.as()) { ASSERT(m.size() == 1); ProjectionSpecByName::instance().regist( m.begin()->first.as(), new SpecByNameGenerator(spec::Custom::make_from_value(m.begin()->second))); } continue; } custom->set(key, kv.second); } } } } // namespace eckit::geo::share eckit-2.0.7/src/eckit/geo/share/Area.h0000664000175000017500000000154315161702250017570 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit { namespace spec { class Spec; } class PathName; } // namespace eckit namespace eckit::geo::share { class Area final { public: static const Area& instance(); private: // -- Constructors explicit Area(const std::vector&); // -- Members std::unique_ptr spec_; // -- Methods void load(const PathName&); }; } // namespace eckit::geo::share eckit-2.0.7/src/eckit/geo/util.h0000664000175000017500000000342215161702250016571 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include #include namespace eckit::geo { using difference_type = std::make_signed_t; using pl_type = std::vector; // sync with eckit::spec::Custom number_type namespace util { constexpr double DEGREE_TO_RADIAN = M_PI / 180.; constexpr double RADIAN_TO_DEGREE = M_1_PI * 180.; template pl_type pl_convert(const T& pl) { ASSERT(!pl.empty()); pl_type _pl(pl.size()); std::transform(pl.begin(), pl.end(), _pl.begin(), [](typename T::value_type p) { return static_cast(p); }); return _pl; } template <> pl_type pl_convert(const pl_type&); std::vector arange(double start, double stop, double step); const std::vector& gaussian_latitudes(size_t N, bool increasing); const std::vector& gaussian_quadrature_weights(size_t N); const std::vector& linspace(double start, double stop, size_t num); std::pair monotonic_crop(const std::vector&, double min, double max); std::vector reverse(const std::vector&); bool reduced_classical_pl_known(size_t N); const pl_type& reduced_classical_pl(size_t N); const pl_type& reduced_octahedral_pl(size_t N); } // namespace util } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/util/0000775000175000017500000000000015161702250016417 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/util/hash_vector.cc0000664000175000017500000000207115161702250021233 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/eckit_config.h" #include "eckit/utils/MD5.h" #if eckit_LITTLE_ENDIAN #else #include "eckit/utils/ByteSwap.h" #endif namespace eckit::geo::util { template void hash_vector(MD5& hash, const std::vector& v) { const auto len = static_cast(v.size() * sizeof(T)); #if eckit_LITTLE_ENDIAN hash.add(v.data(), len); #else auto w = v; byteswap(w); hash.add(w.data(), len); #endif } void hash_vector_double(MD5& hash, const std::vector& v) { hash_vector(hash, v); } void hash_vector_size_t(MD5& hash, const std::vector& v) { hash_vector(hash, v); } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/arange.cc0000664000175000017500000000206415161702250020165 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/types/FloatCompare.h" namespace eckit::geo::util { std::vector arange(double start, double stop, double step) { if (types::is_approximately_equal(step, 0.) || types::is_approximately_equal(start, stop) || (stop - start) * step < 0.) { std::vector l(1, start); return l; } const auto num = static_cast((stop - start) / step) + 1; std::vector l(num); std::generate_n(l.begin(), num, [start, step, n = 0ULL]() mutable { return start + static_cast(n++) * step; }); return l; } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/sincos.h0000664000175000017500000000134215161702250020066 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit::geo::util { struct sincos_t final : std::array { explicit sincos_t(value_type r) : array{std::sin(r), std::cos(r)} {} const value_type& sin = array::operator[](0); const value_type& cos = array::operator[](1); }; } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/reduced_octahedral_pl.cc0000664000175000017500000000160715161702250023226 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/util.h" namespace eckit::geo::util { const pl_type& reduced_octahedral_pl(size_t N) { static cache::MemoryCacheT cache; if (cache.contains(N)) { return cache[N]; } pl_type pl(N * 2); pl_type::value_type p = 20; for (size_t i = 0, j = 2 * N - 1; i < N; ++i, --j) { pl[i] = pl[j] = p; p += 4; } return (cache[N] = std::move(pl)); } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/mutex.h0000664000175000017500000000211515161702250017731 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #define ECKIT_GEO_ECKIT_THREADS #if defined(ECKIT_GEO_ECKIT_THREADS) #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" #else #include #endif namespace eckit::geo::util { #if defined(ECKIT_GEO_ECKIT_THREADS) using recursive_mutex = eckit::Mutex; template using lock_guard = typename eckit::AutoLock; struct once_flag { pthread_once_t once_ = PTHREAD_ONCE_INIT; }; template inline void call_once(once_flag& flag, Callable&& fun) { pthread_once(&(flag.once_), fun); } #else using std::call_once; using std::lock_guard; using std::once_flag; using std::recursive_mutex; #endif } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/monotonic_crop.cc0000664000175000017500000000277315161702250021767 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include "eckit/geo/Exceptions.h" namespace eckit::geo::util { using difference_type = std::make_signed_t; std::pair monotonic_crop(const std::vector& values, double min, double max) { if (values.empty() || min > max) { return {}; } auto b = values.begin(); auto e = values.end(); // monotonically increasing const auto increasing = values.size() == 1 || values.front() < values.back(); if (increasing) { ASSERT(std::is_sorted(b, e)); auto lt = [](double a, double b) { return a < b; }; return {std::distance(b, std::lower_bound(b, e, min, lt)), std::distance(b, std::upper_bound(b, e, max, lt))}; } // monotonically non-increasing ASSERT(std::is_sorted(values.rbegin(), values.rend())); auto gt = [](double a, double b) { return a > b; }; return {std::distance(b, std::lower_bound(b, e, max, gt)), std::distance(b, std::upper_bound(b, e, min, gt))}; } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/linspace.cc0000664000175000017500000000252215161702250020525 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/geo/cache/MemoryCache.h" #include "eckit/types/FloatCompare.h" #include "eckit/utils/MD5.h" namespace eckit::geo::util { const std::vector& linspace(double start, double stop, size_t num) { const auto key = (eckit::MD5{} << start << stop << num).digest(); static cache::MemoryCacheT> cache; if (cache.contains(key)) { return cache[key]; } if (num == 0) { return (cache[key] = {}); } if (num == 1 || types::is_approximately_equal(start, stop)) { return (cache[key] = {start}); } const auto step = (stop - start) / static_cast(num - 1); std::vector l(num); std::generate_n(l.begin(), num, [start, step, n = 0ULL]() mutable { return start + static_cast(n++) * step; }); return (cache[key] = std::move(l)); } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/gaussian_quadrature_weights.cc0000664000175000017500000000700215161702250024526 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/util.h" namespace eckit::geo::util { const std::vector& gaussian_quadrature_weights(size_t N) { ASSERT(N > 0); using cache_t = cache::MemoryCacheT>; const cache_t::key_type key{N}; static cache_t cache; if (cache.contains(key)) { return cache[key]; } std::vector weights(2 * N); // Fourier coefficients of series expansion for the ordinary Legendre polynomials std::vector zzfn(N + 1); { // Belousov, Swarztrauber use zfn(0)=std::sqrt(2.) // IFS normalisation chosen to be 0.5*Integral(Pnm**2) = 1 std::vector zfn(2 * N + 1, 2.); for (size_t i = 1; i <= 2 * N; ++i) { for (size_t j = 1; j <= i; ++j) { zfn[i] *= std::sqrt(1. - 0.25 / (static_cast(j * j))); } for (size_t j = 2; j <= i - (i % 2); j += 2) { zfn[i - j] = zfn[i - j + 2] * static_cast((j - 1) * (2 * i - j + 2)) / static_cast(j * (2 * i - j + 1)); } } for (size_t i = 0; i <= N; ++i) { zzfn[i] = zfn[i * 2]; } } /* * Newton loop (per latitude) to find 0 of Legendre polynomial of degree N (GAWL) * The convergence is run more than once, for historical reasons. */ constexpr size_t Nmax = 20; constexpr auto eps = std::numeric_limits::epsilon() * 1000.; for (size_t i = 0; i < N; ++i) { // First guess for colatitude [rad] auto z = static_cast(4 * (i + 1) - 1) * M_PI / static_cast(4 * 2 * N + 2); auto x = z + 1. / (std::tan(z) * static_cast(8 * (2 * N) * (2 * N))); double w = 0.; auto converged = false; for (size_t n = 1; n < Nmax; ++n) { auto f = zzfn[0] * 0.5 + zzfn[1] * std::cos(2. * x); // normalised ordinary Legendre polynomial == \overbar{P_n}^0 auto fp = zzfn[1] * std::sin(2. * x) * -2.; // normalised derivative == d/d\theta(\overbar{P_n}^0) for (size_t j = 2; j <= N; ++j) { const auto j2 = static_cast(j * 2); f += zzfn[j] * std::cos(j2 * x); fp -= zzfn[j] * std::sin(j2 * x) * j2; } auto dx = -f / fp; x += dx; if (converged) { w = static_cast(2 * 2 * N + 1) / (fp * fp); break; } converged = std::abs(dx) <= eps; } if (!converged) { throw BadValue("Could not calculate latitude within accuracy/iterations: " + std::to_string(eps) + "/" + std::to_string(Nmax), Here()); } // symmetry const auto j = 2 * N - 1 - i; weights[i] = weights[j] = w; } return (cache[key] = std::move(weights)); } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/reduced_classical_pl.cc0000664000175000017500000045422015161702250023061 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/util.h" namespace eckit::geo::util { static const std::map CLASSICAL_PLS{ {16, {20, 27, 32, 40, 45, 48, 60, 60, 64, 64, 64, 64, 64, 64, 64, 64}}, {24, {20, 25, 36, 40, 45, 48, 54, 60, 64, 72, 80, 80, 90, 90, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96}}, {32, {20, 27, 36, 40, 45, 50, 60, 64, 72, 75, 80, 90, 90, 96, 100, 108, 108, 120, 120, 120, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}}, {48, {20, 25, 36, 40, 45, 50, 60, 60, 72, 75, 80, 90, 96, 100, 108, 120, 120, 120, 128, 135, 144, 144, 160, 160, 160, 160, 160, 180, 180, 180, 180, 180, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192}}, {64, {20, 25, 36, 40, 45, 54, 60, 64, 72, 75, 80, 90, 96, 100, 108, 120, 120, 125, 135, 135, 144, 150, 160, 160, 180, 180, 180, 180, 192, 192, 200, 200, 216, 216, 216, 216, 225, 225, 225, 240, 240, 240, 240, 243, 250, 250, 250, 250, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, 256}}, {80, {18, 25, 36, 40, 45, 54, 60, 64, 72, 72, 80, 90, 96, 100, 108, 120, 120, 128, 135, 144, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 200, 216, 216, 216, 225, 225, 240, 240, 240, 256, 256, 256, 256, 288, 288, 288, 288, 288, 288, 288, 288, 288, 300, 300, 300, 300, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320, 320}}, {96, {18, 25, 36, 40, 45, 50, 60, 64, 72, 72, 80, 90, 96, 100, 108, 120, 120, 125, 135, 144, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 200, 216, 216, 225, 225, 240, 240, 240, 250, 250, 256, 270, 270, 270, 288, 288, 288, 288, 300, 300, 300, 320, 320, 320, 320, 320, 324, 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, 375, 375, 375, 375, 375, 375, 375, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, 384}}, {128, {18, 25, 36, 40, 45, 50, 60, 64, 72, 72, 80, 90, 90, 100, 108, 120, 120, 125, 128, 144, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 216, 216, 216, 225, 240, 240, 240, 250, 250, 256, 270, 270, 288, 288, 288, 300, 300, 320, 320, 320, 320, 324, 360, 360, 360, 360, 360, 360, 360, 375, 375, 375, 375, 384, 384, 400, 400, 400, 400, 405, 432, 432, 432, 432, 432, 432, 432, 450, 450, 450, 450, 450, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 486, 486, 486, 500, 500, 500, 500, 500, 500, 500, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512}}, {160, {18, 25, 36, 40, 45, 50, 60, 64, 72, 72, 80, 90, 90, 96, 108, 120, 120, 125, 128, 135, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 216, 216, 225, 225, 240, 240, 243, 250, 256, 270, 270, 288, 288, 288, 300, 300, 320, 320, 320, 320, 324, 360, 360, 360, 360, 360, 360, 375, 375, 375, 384, 384, 400, 400, 400, 405, 432, 432, 432, 432, 432, 450, 450, 450, 450, 480, 480, 480, 480, 480, 480, 480, 500, 500, 500, 500, 500, 512, 512, 540, 540, 540, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, 600, 600, 600, 600, 600, 600, 600, 600, 600, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640}}, {200, {18, 25, 36, 40, 45, 50, 60, 64, 72, 72, 75, 81, 90, 96, 100, 108, 120, 125, 128, 135, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 216, 216, 225, 225, 240, 240, 243, 250, 256, 270, 270, 288, 288, 288, 300, 300, 320, 320, 320, 320, 360, 360, 360, 360, 360, 360, 375, 375, 375, 384, 400, 400, 400, 400, 432, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 480, 480, 486, 500, 500, 500, 512, 512, 512, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 576, 576, 600, 600, 600, 600, 600, 640, 640, 640, 640, 640, 640, 640, 640, 640, 640, 648, 648, 675, 675, 675, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 720, 720, 720, 720, 720, 720, 720, 729, 729, 729, 750, 750, 750, 750, 750, 750, 750, 750, 768, 768, 768, 768, 768, 768, 768, 768, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800, 800}}, {256, {18, 25, 32, 40, 45, 50, 60, 64, 72, 72, 75, 81, 90, 96, 100, 108, 120, 120, 125, 135, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 216, 216, 216, 225, 240, 240, 243, 250, 256, 270, 270, 288, 288, 288, 300, 300, 320, 320, 320, 324, 360, 360, 360, 360, 360, 360, 375, 375, 384, 384, 400, 400, 400, 432, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 480, 486, 500, 500, 500, 512, 512, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 600, 600, 600, 600, 600, 640, 640, 640, 640, 640, 640, 640, 640, 648, 675, 675, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 720, 720, 729, 729, 750, 750, 750, 750, 750, 768, 768, 768, 768, 800, 800, 800, 800, 800, 800, 800, 800, 810, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 972, 972, 972, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024}}, {320, {18, 25, 36, 40, 45, 50, 60, 64, 72, 72, 75, 81, 90, 96, 100, 108, 120, 120, 125, 135, 144, 144, 150, 160, 180, 180, 180, 192, 192, 200, 216, 216, 216, 225, 240, 240, 240, 250, 256, 270, 270, 288, 288, 288, 300, 300, 320, 320, 320, 324, 360, 360, 360, 360, 360, 360, 375, 375, 384, 384, 400, 400, 405, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 480, 486, 500, 500, 500, 512, 512, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 600, 600, 600, 600, 640, 640, 640, 640, 640, 640, 640, 648, 648, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 720, 720, 729, 750, 750, 750, 750, 768, 768, 768, 768, 800, 800, 800, 800, 800, 800, 810, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1215, 1215, 1215, 1215, 1215, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280}}, {400, {18, 25, 32, 40, 45, 50, 60, 60, 72, 72, 75, 81, 90, 96, 100, 108, 120, 120, 125, 128, 144, 144, 150, 160, 160, 180, 180, 192, 192, 200, 200, 216, 216, 225, 240, 240, 240, 250, 250, 256, 270, 288, 288, 288, 300, 300, 320, 320, 320, 324, 360, 360, 360, 360, 360, 360, 375, 375, 384, 400, 400, 400, 405, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 480, 486, 500, 500, 512, 512, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 600, 600, 600, 600, 640, 640, 640, 640, 640, 640, 640, 648, 675, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 729, 729, 750, 750, 750, 750, 768, 768, 768, 800, 800, 800, 800, 800, 800, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 1000, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1215, 1215, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1458, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600}}, {512, {18, 25, 32, 40, 45, 50, 60, 60, 72, 72, 75, 81, 90, 96, 96, 100, 108, 120, 125, 128, 135, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 216, 216, 225, 225, 240, 240, 243, 250, 256, 270, 270, 288, 288, 288, 300, 320, 320, 320, 320, 360, 360, 360, 360, 360, 360, 375, 375, 384, 384, 400, 400, 400, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 480, 486, 500, 500, 512, 512, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 600, 600, 600, 640, 640, 640, 640, 640, 640, 640, 648, 675, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 729, 729, 750, 750, 750, 768, 768, 768, 800, 800, 800, 800, 800, 800, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1215, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 1944, 1944, 1944, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048}}, {576, {18, 25, 32, 40, 45, 50, 60, 60, 72, 72, 75, 81, 90, 96, 96, 100, 108, 120, 120, 125, 135, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 216, 216, 225, 225, 240, 240, 243, 250, 256, 270, 270, 288, 288, 288, 300, 300, 320, 320, 320, 360, 360, 360, 360, 360, 360, 375, 375, 384, 384, 400, 400, 400, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 480, 486, 500, 500, 512, 512, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 600, 600, 600, 600, 640, 640, 640, 640, 640, 640, 640, 648, 675, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 729, 750, 750, 750, 750, 768, 768, 768, 800, 800, 800, 800, 800, 810, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 972, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1215, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304}}, {640, {18, 25, 32, 40, 45, 50, 60, 60, 72, 72, 75, 81, 90, 90, 96, 100, 108, 120, 120, 125, 135, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 216, 216, 216, 225, 240, 240, 243, 250, 256, 270, 270, 288, 288, 288, 300, 300, 320, 320, 320, 360, 360, 360, 360, 360, 360, 375, 375, 384, 384, 400, 400, 400, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 480, 486, 500, 500, 512, 512, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 600, 600, 600, 600, 640, 640, 640, 640, 640, 640, 640, 648, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 720, 729, 750, 750, 750, 750, 768, 768, 768, 800, 800, 800, 800, 800, 810, 810, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1215, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560}}, {800, {18, 25, 32, 40, 45, 50, 60, 60, 72, 72, 75, 80, 90, 90, 96, 100, 108, 120, 120, 125, 128, 135, 144, 150, 160, 160, 180, 180, 192, 192, 200, 200, 216, 216, 225, 240, 240, 240, 250, 250, 256, 270, 288, 288, 288, 300, 300, 320, 320, 320, 324, 360, 360, 360, 360, 360, 375, 375, 375, 384, 400, 400, 400, 405, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 486, 500, 500, 500, 512, 512, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 600, 600, 600, 625, 625, 625, 625, 625, 640, 640, 648, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 720, 729, 750, 750, 750, 768, 768, 768, 800, 800, 800, 800, 800, 800, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1250, 1250, 1250, 1250, 1250, 1250, 1280, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2048, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2187, 2187, 2187, 2187, 2187, 2187, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2592, 2592, 2592, 2592, 2592, 2592, 2592, 2592, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200}}, {1024, {18, 25, 32, 40, 45, 50, 60, 64, 72, 72, 75, 81, 90, 96, 96, 108, 108, 120, 120, 125, 125, 135, 144, 150, 160, 160, 180, 180, 180, 192, 192, 200, 216, 216, 225, 225, 240, 240, 243, 250, 256, 270, 270, 288, 288, 288, 300, 300, 320, 320, 320, 360, 360, 360, 360, 360, 360, 375, 375, 384, 384, 400, 400, 405, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 480, 486, 500, 500, 512, 512, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 600, 600, 600, 600, 625, 625, 625, 625, 640, 640, 648, 675, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 729, 750, 750, 750, 750, 768, 768, 800, 800, 800, 800, 800, 800, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1215, 1250, 1250, 1250, 1250, 1250, 1250, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2048, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2187, 2187, 2187, 2187, 2187, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2430, 2430, 2430, 2430, 2430, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2592, 2592, 2592, 2592, 2592, 2592, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096}}, {1280, {18, 25, 32, 40, 45, 50, 60, 64, 72, 72, 75, 81, 90, 96, 96, 108, 108, 120, 120, 120, 125, 135, 135, 144, 144, 160, 160, 180, 180, 180, 192, 200, 200, 216, 216, 225, 240, 240, 240, 250, 250, 256, 270, 288, 288, 288, 300, 300, 320, 320, 320, 324, 360, 360, 360, 360, 360, 375, 375, 375, 384, 400, 400, 400, 432, 432, 432, 432, 432, 450, 450, 480, 480, 480, 480, 480, 486, 500, 500, 512, 512, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 600, 600, 600, 600, 625, 625, 625, 625, 640, 640, 640, 648, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 729, 729, 750, 750, 750, 768, 768, 768, 800, 800, 800, 800, 800, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1250, 1250, 1250, 1250, 1250, 1250, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2187, 2187, 2187, 2187, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2430, 2430, 2430, 2430, 2430, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2592, 2592, 2592, 2592, 2592, 2592, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2916, 2916, 2916, 2916, 2916, 2916, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120}}, {1600, {18, 25, 32, 40, 45, 50, 54, 60, 72, 72, 75, 80, 90, 90, 96, 100, 108, 120, 120, 120, 125, 128, 135, 144, 144, 150, 160, 160, 162, 180, 180, 180, 192, 192, 216, 216, 225, 240, 240, 243, 250, 256, 270, 270, 288, 288, 288, 300, 300, 320, 320, 320, 360, 360, 360, 360, 360, 360, 375, 375, 384, 384, 400, 400, 405, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 480, 486, 500, 500, 512, 512, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 600, 600, 600, 600, 625, 625, 625, 625, 640, 640, 648, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 729, 729, 750, 750, 750, 768, 768, 768, 800, 800, 800, 800, 800, 810, 810, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1250, 1250, 1250, 1250, 1250, 1250, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2187, 2187, 2187, 2187, 2187, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2430, 2430, 2430, 2430, 2430, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2592, 2592, 2592, 2592, 2592, 2592, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2916, 2916, 2916, 2916, 2916, 2916, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400}}, {2000, {18, 25, 32, 40, 45, 50, 60, 60, 72, 72, 75, 81, 90, 96, 96, 108, 108, 120, 120, 120, 125, 135, 135, 144, 144, 150, 160, 160, 180, 180, 180, 180, 192, 192, 192, 200, 216, 216, 216, 225, 240, 240, 243, 250, 256, 270, 270, 288, 300, 300, 320, 320, 320, 360, 360, 360, 360, 360, 360, 375, 375, 384, 400, 400, 400, 405, 432, 432, 432, 432, 450, 450, 450, 480, 480, 480, 480, 486, 500, 500, 500, 512, 512, 540, 540, 540, 540, 576, 576, 576, 576, 576, 600, 600, 600, 600, 625, 625, 625, 625, 640, 640, 640, 648, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 729, 750, 750, 750, 750, 768, 768, 768, 800, 800, 800, 800, 800, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1215, 1250, 1250, 1250, 1250, 1250, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2187, 2187, 2187, 2187, 2187, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2430, 2430, 2430, 2430, 2430, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2592, 2592, 2592, 2592, 2592, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2916, 2916, 2916, 2916, 2916, 2916, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000}}, {4000, {18, 24, 32, 40, 45, 48, 54, 60, 64, 72, 75, 80, 90, 90, 96, 100, 108, 108, 120, 120, 125, 128, 135, 135, 144, 150, 150, 160, 160, 180, 180, 180, 180, 192, 192, 192, 200, 216, 216, 216, 216, 225, 225, 240, 240, 240, 243, 250, 256, 256, 270, 270, 270, 288, 288, 288, 288, 300, 300, 300, 320, 320, 320, 320, 324, 360, 360, 360, 360, 360, 360, 360, 360, 375, 375, 375, 375, 384, 384, 400, 400, 400, 405, 405, 432, 432, 432, 432, 432, 432, 450, 450, 450, 450, 480, 480, 480, 480, 480, 480, 486, 486, 500, 500, 500, 512, 512, 512, 540, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 576, 576, 600, 600, 600, 600, 600, 600, 625, 625, 625, 625, 625, 625, 640, 640, 640, 648, 648, 675, 675, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 720, 720, 720, 720, 729, 729, 750, 750, 750, 750, 750, 768, 768, 768, 768, 800, 800, 800, 800, 800, 800, 800, 810, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 972, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1215, 1215, 1250, 1250, 1250, 1250, 1250, 1250, 1250, 1250, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2048, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2430, 2430, 2430, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2592, 2592, 2592, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2916, 2916, 2916, 2916, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3240, 3240, 3240, 3240, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3645, 3645, 3645, 3645, 3645, 3645, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3888, 3888, 3888, 3888, 3888, 3888, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4050, 4050, 4050, 4050, 4050, 4050, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 4860, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000}}, {8000, {16, 24, 30, 36, 40, 45, 54, 60, 64, 72, 72, 75, 81, 90, 90, 96, 100, 108, 120, 120, 120, 125, 128, 135, 144, 144, 150, 160, 160, 160, 180, 180, 180, 180, 192, 192, 192, 200, 216, 216, 216, 216, 225, 225, 240, 240, 240, 243, 250, 250, 256, 270, 270, 270, 288, 288, 288, 288, 300, 300, 300, 320, 320, 320, 320, 324, 360, 360, 360, 360, 360, 360, 360, 360, 375, 375, 375, 375, 384, 384, 400, 400, 400, 405, 405, 432, 432, 432, 432, 432, 432, 450, 450, 450, 450, 480, 480, 480, 480, 480, 480, 480, 486, 500, 500, 500, 512, 512, 512, 540, 540, 540, 540, 540, 540, 540, 576, 576, 576, 576, 576, 576, 576, 576, 600, 600, 600, 600, 600, 625, 625, 625, 625, 625, 625, 640, 640, 640, 640, 648, 675, 675, 675, 675, 675, 675, 675, 720, 720, 720, 720, 720, 720, 720, 720, 720, 720, 729, 729, 750, 750, 750, 750, 750, 768, 768, 768, 768, 800, 800, 800, 800, 800, 800, 800, 800, 810, 810, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 864, 900, 900, 900, 900, 900, 900, 900, 900, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 960, 972, 972, 972, 1000, 1000, 1000, 1000, 1000, 1000, 1024, 1024, 1024, 1024, 1024, 1024, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1080, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1152, 1152, 1152, 1152, 1152, 1152, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1200, 1215, 1215, 1215, 1215, 1250, 1250, 1250, 1250, 1250, 1250, 1250, 1250, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1296, 1296, 1296, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1350, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1440, 1458, 1458, 1458, 1458, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1536, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1600, 1620, 1620, 1620, 1620, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1728, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1875, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1920, 1944, 1944, 1944, 1944, 1944, 1944, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2025, 2025, 2025, 2025, 2025, 2025, 2048, 2048, 2048, 2048, 2048, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2160, 2187, 2187, 2187, 2187, 2187, 2187, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2250, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2304, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2430, 2430, 2430, 2430, 2430, 2430, 2430, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2560, 2592, 2592, 2592, 2592, 2592, 2592, 2592, 2592, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2700, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2880, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 2916, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3000, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3072, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3125, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3200, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3240, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3375, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3456, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3600, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3645, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3750, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3840, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 3888, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4000, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4050, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4320, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4374, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4500, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4608, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4800, 4860, 4860, 4860, 4860, 4860, 4860, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5120, 5184, 5184, 5184, 5184, 5184, 5184, 5184, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5400, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5625, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5760, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 5832, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6075, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6144, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6250, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6400, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6480, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6561, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6750, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 6912, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7200, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7290, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7500, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7680, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 7776, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8000, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8100, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8640, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 8748, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9000, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9216, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9375, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9600, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 9720, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10000, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10125, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10240, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10368, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10800, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 10935, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11250, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11520, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 11664, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12000, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12150, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12288, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12500, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12800, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 12960, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13122, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13500, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 13824, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14400, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 14580, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15000, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15360, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15552, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 15625, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16000, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16200, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 16875, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17280, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 17496, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18000, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18225, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18432, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 18750, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19200, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19440, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 19683, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20000, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20250, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20480, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 20736, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21600, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 21870, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 22500, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23040, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 23328, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24000, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24300, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 24576, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25000, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25600, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 25920, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 26244, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27000, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 27648, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28125, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 28800, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 29160, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30000, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30375, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 30720, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31104, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 31250, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000, 32000}}, }; bool reduced_classical_pl_known(size_t N) { return CLASSICAL_PLS.find(N) != CLASSICAL_PLS.end(); } const pl_type& reduced_classical_pl(size_t N) { ASSERT(N > 0); static cache::MemoryCacheT cache; if (cache.contains(N)) { return cache[N]; } auto pl_half = CLASSICAL_PLS.find(N); if (pl_half == CLASSICAL_PLS.end()) { throw BadValue("reduced_classical_pl: unknown N=" + std::to_string(N), Here()); } ASSERT(pl_half->second.size() == N); pl_type pl(N * 2); auto p = pl_half->second.begin(); for (size_t i = 0, j = 2 * N - 1; i < N; ++i, --j) { pl[i] = pl[j] = *p++; } return (cache[N] = std::move(pl)); } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/gaussian_latitudes.cc0000664000175000017500000000712415161702250022622 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/cache/MemoryCache.h" #include "eckit/geo/util.h" namespace eckit::geo::util { const std::vector& gaussian_latitudes(size_t N, bool increasing) { ASSERT(N > 0); using cache_t = cache::MemoryCacheT, std::vector>; const cache_t::key_type key{N, increasing}; static cache_t cache; if (cache.contains(key)) { return cache[key]; } std::vector lats(2 * N); // Fourier coefficients of series expansion for the ordinary Legendre polynomials std::vector zzfn(N + 1); { // Belousov, Swarztrauber use zfn(0)=std::sqrt(2.) // IFS normalisation chosen to be 0.5*Integral(Pnm**2) = 1 std::vector zfn(2 * N + 1, 2.); for (size_t i = 1; i <= 2 * N; ++i) { for (size_t j = 1; j <= i; ++j) { zfn[i] *= std::sqrt(1. - 0.25 / (static_cast(j * j))); } for (size_t j = 2; j <= i - (i % 2); j += 2) { zfn[i - j] = zfn[i - j + 2] * static_cast((j - 1) * (2 * i - j + 2)) / static_cast(j * (2 * i - j + 1)); } } for (size_t i = 0; i <= N; ++i) { zzfn[i] = zfn[i * 2]; } } /* * Newton loop (per latitude) to find 0 of Legendre polynomial of degree N (GAWL) * The convergence is run more than once, for historical reasons. */ constexpr size_t Nmax = 20; constexpr auto eps = std::numeric_limits::epsilon() * 1000.; for (size_t i = 0; i < N; ++i) { // First guess for colatitude [rad] auto z = static_cast(4 * (i + 1) - 1) * M_PI / static_cast(4 * 2 * N + 2); auto x = z + 1. / (std::tan(z) * static_cast(8 * (2 * N) * (2 * N))); auto converged = false; for (size_t n = 0; n < Nmax; ++n) { auto f = zzfn[0] * 0.5 + zzfn[1] * std::cos(2. * x); // normalised ordinary Legendre polynomial == \overbar{P_n}^0 auto fp = zzfn[1] * std::sin(2. * x) * -2.; // normalised derivative == d/d\theta(\overbar{P_n}^0) for (size_t j = 2; j <= N; ++j) { const auto j2 = static_cast(j * 2); f += zzfn[j] * std::cos(j2 * x); fp -= zzfn[j] * std::sin(j2 * x) * j2; } auto dx = -f / fp; x += dx; if (converged) { break; } converged = std::abs(dx) <= eps; } if (!converged) { throw BadValue("Could not calculate latitude within accuracy/iterations: " + std::to_string(eps) + "/" + std::to_string(Nmax), Here()); } // Convert colatitude [rad] to latitude [degree], symmetry const auto j = 2 * N - 1 - i; lats[i] = (increasing ? (x - M_PI_2) : (M_PI_2 - x)) * RADIAN_TO_DEGREE; lats[j] = -lats[i]; } return (cache[key] = std::move(lats)); } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/util/reverse.cc0000664000175000017500000000125415161702250020403 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include namespace eckit::geo::util { std::vector reverse(const std::vector& v) { std::vector flipped(v.size()); std::reverse_copy(v.begin(), v.end(), flipped.begin()); return v; } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/Shape.cc0000664000175000017500000000225615161702250017016 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Shape.h" #include "eckit/geo/Exceptions.h" #include "eckit/spec/Spec.h" namespace eckit::geo { Shape Shape::make_from_spec(const spec::Spec& spec) { if (std::vector shape; spec.get("shape", shape) && shape.size() == 2) { return {shape[0], shape[1]}; } if (value_type nx = 0, ny = 0; (spec.get("nlon", nx) && spec.get("nlat", ny)) || (spec.get("nlon", nx) && spec.get("nlat", ny))) { return {nx, ny}; } throw exception::SpecError("'shape' = ['nlon', 'nlat'] = ['nx', 'ny'] expected", Here()); } Shape::Shape(value_type nx, value_type ny) : array{nx, ny} { if (!(nx > 0) || !(ny > 0)) { throw BadValue("'shape' = ['nlon', 'nlat'] = ['nx', 'ny'] > 0 expected", Here()); } } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Grid.h0000664000175000017500000001602015161702250016477 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include #include "eckit/geo/Area.h" #include "eckit/geo/Iterator.h" #include "eckit/geo/Point.h" #include "eckit/geo/Projection.h" #include "eckit/geo/area/BoundingBox.h" #include "eckit/memory/Builder.h" #include "eckit/memory/Factory.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Generator.h" namespace eckit { class JSON; namespace geo { class Area; } } // namespace eckit namespace eckit::geo { class Grid { public: // -- Types using uid_type = std::string; using order_type = std::string; using renumber_type = std::vector; using BoundingBox = area::BoundingBox; using builder_t = BuilderT1; using Spec = spec::Spec; using ARG1 = const Spec&; struct Iterator final : std::shared_ptr { explicit Iterator(geo::Iterator* it) : shared_ptr(it) { ASSERT(shared_ptr::operator bool()); } using iterator_category = element_type::iterator_category; using difference_type = element_type::difference_type; using value_type = element_type::value_type; using pointer = element_type::pointer; using reference = element_type::reference; bool operator==(const Iterator& other) const { return get()->operator==(*(other.get())); } bool operator!=(const Iterator& other) const { return get()->operator!=(*(other.get())); } bool operator++() { return get()->operator++(); } bool operator+=(difference_type d) { return get()->operator+=(d); } bool operator--() { return get()->operator--(); } bool operator-=(difference_type d) { return get()->operator-=(d); } explicit operator bool() const { return get()->operator bool(); } Point operator*() const { return get()->operator*(); } size_t index() const { return get()->index(); } }; using iterator = Iterator; // -- Constructors Grid(const Grid&) = delete; Grid(Grid&&) = delete; // -- Destructor virtual ~Grid() = default; // -- Operators Grid& operator=(const Grid&) = delete; Grid& operator=(Grid&&) = delete; // -- Methods iterator begin() const { return cbegin(); } iterator end() const { return cend(); } virtual iterator cbegin() const = 0; virtual iterator cend() const = 0; [[nodiscard]] const spec::Spec& catalog() const; [[nodiscard]] const Spec& spec() const; std::string spec_str() const { return spec().str(); } virtual const std::string& type() const = 0; virtual std::vector shape() const = 0; virtual bool empty() const; virtual size_t size() const; uid_type uid() const; [[nodiscard]] virtual uid_type calculate_uid() const; static bool is_uid(const std::string& uid); virtual bool includesNorthPole() const; virtual bool includesSouthPole() const; virtual bool isPeriodicWestEast() const; [[nodiscard]] virtual Point first_point() const; [[nodiscard]] virtual Point last_point() const; [[nodiscard]] virtual std::vector to_points() const; [[nodiscard]] virtual std::pair, std::vector> to_latlons() const; virtual const order_type& order() const; virtual renumber_type reorder(const order_type&) const; virtual const Area& area() const; virtual renumber_type crop(const Area&) const; virtual const Projection& projection() const; virtual const BoundingBox& boundingBox() const; [[nodiscard]] virtual BoundingBox* calculate_bbox() const; [[nodiscard]] virtual Grid* make_grid_reordered(const order_type&) const; [[nodiscard]] virtual Grid* make_grid_cropped(const Area&) const; // -- Class methods static std::string className() { return "grid"; } protected: // -- Constructors explicit Grid(Projection* = nullptr); // -- Methods virtual void fill_spec(spec::Custom&) const; void reset_uid(uid_type = {}); void projection(Projection* ptr) { projection_.reset(ptr); } void boundingBox(BoundingBox* bbox) { bbox_.reset(bbox); } private: // -- Members mutable std::unique_ptr bbox_; mutable std::unique_ptr projection_; mutable std::unique_ptr catalog_; mutable std::unique_ptr spec_; mutable uid_type uid_; // -- Friends friend bool operator==(const Grid& a, const Grid& b) { return a.spec_str() == b.spec_str(); } friend bool operator!=(const Grid& a, const Grid& b) { return !(a == b); } }; struct GridSpecByName { using key_t = std::string; using generator_t = spec::GeneratorT>; using concrete_generator_t = generator_t::generator_t; static generator_t& instance(); static void regist(const key_t& key, concrete_generator_t* gen) { generator_t::instance().regist(key, gen); } static void unregist(const key_t& key) { generator_t::instance().unregist(key); } }; struct GridSpecByUID { using key_t = Grid::uid_type; using generator_t = spec::GeneratorT; using concrete_generator_t = generator_t::generator_t; static generator_t& instance(); static void regist(const key_t& key, concrete_generator_t* gen) { generator_t::instance().regist(key, gen); } static void unregist(const key_t& key) { generator_t::instance().unregist(key); } }; template using GridRegisterType = ConcreteBuilderT1; template using GridRegisterUID = spec::ConcreteSpecGeneratorT0; template bool GridRegisterName(const std::string& name_or_pattern) { new eckit::spec::ConcreteSpecGeneratorT1(name_or_pattern); return true; }; struct GridFactory { // This is 'const' as Grid should always be immutable [[nodiscard]] static const Grid* build(const Grid::Spec& spec) { return instance().make_from_spec_(spec); } // This is 'const' as Grid should always be immutable [[nodiscard]] static const Grid* make_from_string(const std::string&); [[nodiscard]] static Grid::Spec* make_spec(const Grid::Spec& spec) { return instance().make_spec_(spec); } static std::ostream& list(std::ostream& out) { return instance().list_(out); } private: static GridFactory& instance(); // This is 'const' as Grid should always be immutable [[nodiscard]] const Grid* make_from_spec_(const Grid::Spec&) const; [[nodiscard]] Grid::Spec* make_spec_(const Grid::Spec&) const; std::ostream& list_(std::ostream&) const; }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/GreatCircle.cc0000664000175000017500000001105215161702250020134 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/GreatCircle.h" #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/util.h" #include "eckit/geo/util/sincos.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo { using types::is_approximately_equal; static bool is_pole(const double lat) { return is_approximately_equal(std::abs(lat), 90.); } GreatCircle::GreatCircle(const PointLonLat& Alonlat, const PointLonLat& Blonlat) : A_(Alonlat), B_(Blonlat) { const bool Apole = is_pole(A_.lat()); const bool Bpole = is_pole(B_.lat()); const double lon12_deg = PointLonLat::normalise_angle_to_minimum(A_.lon() - B_.lon(), -PointLonLat::FLAT_ANGLE); const bool lon_same = Apole || Bpole || is_approximately_equal(lon12_deg, 0.); const bool lon_opposite = Apole || Bpole || is_approximately_equal(std::abs(lon12_deg), 180.); const bool lat_same = is_approximately_equal(A_.lat(), B_.lat()); const bool lat_opposite = is_approximately_equal(A_.lat(), -B_.lat()); if ((lat_same && lon_same) || (lat_opposite && lon_opposite)) { std::ostringstream oss; oss.precision(std::numeric_limits::max_digits10); oss << "Great circle cannot be defined by points collinear with the centre, " << A_ << " and " << B_; throw BadValue(oss.str(), Here()); } crossesPoles_ = lon_same || lon_opposite; } std::vector GreatCircle::latitude(double lon) const { if (crossesPoles()) { return {}; } const double lat1 = util::DEGREE_TO_RADIAN * A_.lat(); const double lat2 = util::DEGREE_TO_RADIAN * B_.lat(); const double lambda1p = util::DEGREE_TO_RADIAN * (lon - A_.lon()); const double lambda2p = util::DEGREE_TO_RADIAN * (lon - B_.lon()); const double lambda = util::DEGREE_TO_RADIAN * PointLonLat::normalise_angle_to_minimum(B_.lon() - A_.lon(), -PointLonLat::FLAT_ANGLE); double lat = std::atan((std::tan(lat2) * std::sin(lambda1p) - std::tan(lat1) * std::sin(lambda2p)) / (std::sin(lambda))); return {util::RADIAN_TO_DEGREE * lat}; } std::vector GreatCircle::longitude(double lat) const { if (crossesPoles()) { const double lon = is_pole(A_.lat()) ? B_.lon() : A_.lon(); if (is_pole(lat)) { return {lon}; } return {lon, lon + 180.}; } const double lon12 = util::DEGREE_TO_RADIAN * PointLonLat::normalise_angle_to_minimum(A_.lon() - B_.lon(), -PointLonLat::FLAT_ANGLE); const double lon1 = util::DEGREE_TO_RADIAN * A_.lon(); const double lat1 = util::DEGREE_TO_RADIAN * A_.lat(); const double lat2 = util::DEGREE_TO_RADIAN * B_.lat(); const double lat3 = util::DEGREE_TO_RADIAN * lat; const double X = std::sin(lat1) * std::cos(lat2) * std::sin(lon12); const double Y = std::sin(lat1) * std::cos(lat2) * std::cos(lon12) - std::cos(lat1) * std::sin(lat2); if (is_approximately_equal(X, 0.) && is_approximately_equal(Y, 0.)) { return {}; // parallel (that is, equator) } const double lon0 = lon1 + atan2(Y, X); const double C = std::cos(lat1) * std::cos(lat2) * std::tan(lat3) * std::sin(lon12) / std::sqrt(X * X + Y * Y); if (is_approximately_equal(C, -1.)) { return {util::RADIAN_TO_DEGREE * (lon0 + M_PI)}; } if (is_approximately_equal(C, 1.)) { return {util::RADIAN_TO_DEGREE * lon0}; } if (-1 < C && C < 1) { const double dlon = std::acos(C); return {util::RADIAN_TO_DEGREE * (lon0 - dlon + 2 * M_PI), util::RADIAN_TO_DEGREE * (lon0 + dlon)}; } return {}; } bool GreatCircle::crossesPoles() const { return crossesPoles_; } std::pair GreatCircle::course() const { const util::sincos_t dl(util::DEGREE_TO_RADIAN * (B_.lon() - A_.lon())); const util::sincos_t scA(util::DEGREE_TO_RADIAN * A_.lat()); const util::sincos_t scB(util::DEGREE_TO_RADIAN * B_.lat()); return {util::RADIAN_TO_DEGREE * std::atan2(scB.cos * dl.sin, scA.cos * scB.sin - scA.sin * scB.cos * dl.cos), util::RADIAN_TO_DEGREE * std::atan2(scA.cos * dl.sin, -scB.cos * scA.sin + scB.sin * scA.cos * dl.cos)}; } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Trace.cc0000664000175000017500000000201715161702250017007 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Trace.h" #include "eckit/config/Resource.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/log/ResourceUsage.h" namespace eckit::geo { Trace::Trace(const std::string& name) : Timer(name, Log::debug()) {} TraceResourceUsage::TraceResourceUsage(const std::string& name) : Trace(name) { static bool usage = LibResource( "eckit-geo-trace-resource-usage;" "$ECKIT_GEO_TRACE_RESOURCE_USAGE", false); info_ = usage ? new eckit::ResourceUsage(name, Log::debug()) : nullptr; } TraceResourceUsage::~TraceResourceUsage() { delete info_; } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Container.h0000664000175000017500000000157515161702250017545 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include namespace eckit::geo { template class Container { protected: Container() = default; public: virtual ~Container() = default; Container(const Container&) = delete; Container(Container&&) = delete; Container& operator=(const Container&) = delete; Container& operator=(Container&&) = delete; virtual T get(size_t index) const = 0; virtual size_t size() const = 0; }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/LibEcKitGeo.h0000664000175000017500000000225515161702250017700 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/system/Library.h" namespace eckit { class PathName; } namespace eckit { class LibEcKitGeo final : public system::Library { public: // -- Methods static LibEcKitGeo& instance(); static std::vector shareArea(); static std::vector shareGrid(); static std::vector shareProjection(); static bool caching(); static std::string cacheDir(); static std::string url(const std::string& url_path); static bool proj(); private: // -- Constructors LibEcKitGeo(); // -- Overridden methods [[nodiscard]] const void* addr() const override; std::string version() const override; std::string gitsha1(unsigned int count) const override; }; } // namespace eckit eckit-2.0.7/src/eckit/geo/PointXY.h0000664000175000017500000000656215161702250017176 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit::geo { /** * @brief The PointXY class * @details A point on two-dimensional space, in (X, Y) coordinates, linear in space. */ class PointXY final : protected std::array { public: // -- Types using container_type = array; using container_type::value_type; // -- Constructors PointXY() : PointXY(0., 0.) {} PointXY(value_type x, value_type y) : container_type{x, y} {} PointXY(const PointXY& other) : container_type(other) {} PointXY(PointXY&& other) : container_type(other) {} // -- Destructor ~PointXY() = default; // -- Operators using container_type::operator[]; PointXY& operator=(const PointXY& other) { container_type::operator=(other); return *this; } PointXY& operator=(PointXY&& other) { container_type::operator=(other); return *this; } // -- Members value_type X() const { return container_type::operator[](0); } value_type Y() const { return container_type::operator[](1); } // -- Methods using container_type::data; static size_t dimensions() { return DIMS; } static value_type norm(const PointXY& p) { return p.norm(); } static PointXY normalize(const PointXY& p) { return p.normalize(); } static PointXY middle(const PointXY& p, const PointXY& q) { return p.middle(q); } static value_type distance(const PointXY& p, const PointXY& q, size_t axis) { return p.distance(q, axis); } static value_type distance(const PointXY& p, const PointXY& q) { return p.distance(q); } static value_type distance2(const PointXY& p, const PointXY& q) { return p.distance2(q); } value_type norm() const; PointXY normalize() const; PointXY middle(const PointXY&) const; value_type distance(const PointXY&, size_t axis) const; value_type distance(const PointXY&) const; value_type distance2(const PointXY&) const; value_type x(size_t axis) const { return container_type::operator[](axis); } // -- Class members static constexpr size_t DIMS = 2; static constexpr value_type EPS = 1e-9; // -- Friends friend std::ostream& operator<<(std::ostream& out, const PointXY& p) { return out << '{' << p.X() << ", " << p.Y() << '}'; } friend PointXY operator-(const PointXY& p) { return {-p.X(), -p.Y()}; } friend PointXY operator-(const PointXY& p, const PointXY& q) { return {p.X() - q.X(), p.Y() - q.Y()}; } friend PointXY operator+(const PointXY& p, const PointXY& q) { return {p.X() + q.X(), p.Y() + q.Y()}; } friend PointXY operator*(const PointXY& p, value_type d) { return {p.X() * d, p.Y() * d}; } friend bool operator==(const PointXY& p, const PointXY& q) { return p.X() == q.X() && p.Y() == q.Y(); } friend bool operator!=(const PointXY& p, const PointXY& q) { return !operator==(p, q); } }; bool points_equal(const PointXY&, const PointXY&, PointXY::value_type eps = PointXY::EPS); } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/eckit_geo_config.h.in0000664000175000017500000000245515161702250021504 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #cmakedefine01 eckit_HAVE_GEO_AREA_SHAPEFILE constexpr bool _to_bool(int x) { return x != 0; } constexpr bool eckit_HAVE_ECKIT_CODEC = _to_bool(@eckit_HAVE_ECKIT_CODEC@); #cmakedefine01 eckit_HAVE_PROJ constexpr bool eckit_HAVE_GEO_CACHING = _to_bool(@eckit_HAVE_GEO_CACHING@); constexpr bool eckit_HAVE_GEO_BITREPRODUCIBLE = _to_bool(@eckit_HAVE_GEO_BITREPRODUCIBLE@); constexpr bool eckit_HAVE_GEO_PROJECTION_PROJ_DEFAULT = _to_bool(@eckit_HAVE_GEO_PROJECTION_PROJ_DEFAULT@); constexpr const char* eckit_GEO_CACHE_PATH = "@eckit_GEO_CACHE_PATH@"; constexpr const char* eckit_GEO_SHARE_URL_PREFIX = "@eckit_GEO_SHARE_URL_PREFIX@"; constexpr const char* eckit_GEO_SHARE_AREA = "@eckit_GEO_SHARE_AREA@"; constexpr const char* eckit_GEO_SHARE_GRID = "@eckit_GEO_SHARE_GRID@"; constexpr const char* eckit_GEO_SHARE_PROJECTION = "@eckit_GEO_SHARE_PROJECTION@"; eckit-2.0.7/src/eckit/geo/order/0000775000175000017500000000000015161702250016555 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/order/HEALPix.cc0000664000175000017500000001672215161702250020266 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/order/HEALPix.h" #include #include #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/grid/reduced/HEALPix.h" #include "eckit/spec/Spec.h" namespace eckit::geo::order { const HEALPix::order_type HEALPix::RING = "ring"; const HEALPix::order_type HEALPix::NESTED = "nested"; namespace { struct CodecFijNest { static constexpr uint64_t MASKS[] = {0x00000000ffffffff, 0x0000ffff0000ffff, 0x00ff00ff00ff00ff, 0x0f0f0f0f0f0f0f0f, 0x3333333333333333, 0x5555555555555555}; inline static int nest_encode_bits(int n) { auto b = static_cast(n) & MASKS[0]; b = (b ^ (b << 16)) & MASKS[1]; b = (b ^ (b << 8)) & MASKS[2]; b = (b ^ (b << 4)) & MASKS[3]; b = (b ^ (b << 2)) & MASKS[4]; b = (b ^ (b << 1)) & MASKS[5]; return static_cast(b); } inline static int nest_decode_bits(int n) { auto b = static_cast(n) & MASKS[5]; b = (b ^ (b >> 1)) & MASKS[4]; b = (b ^ (b >> 2)) & MASKS[3]; b = (b ^ (b >> 4)) & MASKS[2]; b = (b ^ (b >> 8)) & MASKS[1]; b = (b ^ (b >> 16)) & MASKS[0]; return static_cast(b); } static std::tuple nest_to_fij(int n, int k) { ASSERT(0 <= n); auto f = n >> (2 * k); // f = n / (Nside * Nside) n &= (1 << (2 * k)) - 1; // n = n % (Nside * Nside) auto i = nest_decode_bits(n); auto j = nest_decode_bits(n >> 1); return {f, i, j}; } static int fij_to_nest(int f, int i, int j, int k) { return (f << (2 * k)) + nest_encode_bits(i) + (nest_encode_bits(j) << 1); } }; inline int sqrt(int n) { return static_cast(std::sqrt(static_cast(n) + 0.5)); } // for division result within [0; 3] inline int div_03(int a, int b) { int t = (a >= (b << 1)) ? 1 : 0; a -= t * (b << 1); return (t << 1) + (a >= b ? 1 : 0); } inline bool is_power_of_2(int n) { return std::bitset(n).count() == 1; } inline int pll(int f) { constexpr int __pll[] = {1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7}; return __pll[f]; } struct Renumber { const int Nside_; // up to 2^13 const int Npix_; const int Ncap_; const int k_; Renumber(int Nside) : Nside_(Nside), Npix_(static_cast(grid::reduced::HEALPix::size_from_nside(Nside_))), Ncap_((Nside_ * (Nside_ - 1)) << 1), k_(static_cast(std::log2(Nside_))) { ASSERT_MSG(is_power_of_2(Nside_), "Nside must be a power of 2"); ASSERT(0 <= k_); } int ring_to_nest(int r) const { auto to_nest = [&](int f, //!< base pixel index int ring, //!< 1-based ring number int Nring, //!< number of pixels in ring int phi, //!< index in longitude int shift //!< if ring's first pixel is not at phi=0 ) -> int { int r = ((2 + (f >> 2)) << k_) - ring - 1; int p = 2 * phi - pll(f) * Nring - shift - 1; if (p >= 2 * Nside_) { p -= 8 * Nside_; } int i = (r + p) >> 1; int j = (r - p) >> 1; ASSERT(f < 12 && i < Nside_ && j < Nside_); return CodecFijNest::fij_to_nest(f, i, j, k_); }; if (Nside_ == 1) { return r; } if (r < Ncap_) { // North polar cap int Nring = (1 + sqrt(2 * r + 1)) >> 1; int phi = 1 + r - 2 * Nring * (Nring - 1); int r = Nring; int f = div_03(phi - 1, Nring); return to_nest(f, r, Nring, phi, 0); } if (Npix_ - Ncap_ <= r) { // South polar cap int Nring = (1 + sqrt(2 * Npix_ - 2 * r - 1)) >> 1; int phi = 1 + r + 2 * Nring * (Nring - 1) + 4 * Nring - Npix_; int ring = 4 * Nside_ - Nring; // (from South pole) int f = div_03(phi - 1, Nring) + 8; return to_nest(f, ring, Nring, phi, 0); } // Equatorial belt int ip = r - Ncap_; int tmp = ip >> (k_ + 2); int Nring = Nside_; int phi = ip - tmp * 4 * Nside_ + 1; int ring = tmp + Nside_; int ifm = 1 + ((phi - 1 - ((1 + tmp) >> 1)) >> k_); int ifp = 1 + ((phi - 1 - ((1 - tmp + 2 * Nside_) >> 1)) >> k_); int f = (ifp == ifm) ? (ifp | 4) : ((ifp < ifm) ? ifp : (ifm + 8)); return to_nest(f, ring, Nring, phi, ring & 1); } int nest_to_ring(int n) const { auto [f, i, j] = CodecFijNest::nest_to_fij(n, k_); ASSERT(f < 12 && i < Nside_ && j < Nside_); auto to_ring_local = [&](int f, int i, int j, int Nring, //!< number of pixels in ring int shift //!< if ring's first pixel is/is not at phi=0 ) -> int { Nring >>= 2; int r = (pll(f) * Nring + i - j + 1 + shift) / 2 - 1; ASSERT(r < 4 * Nring); return r < 0 ? r + 4 * Nside_ : r; }; const int ring = ((f >> 2) + 2) * Nside_ - i - j - 1; // 1-based ring number if (ring < Nside_) { // North polar cap int Nring = 4 * ring; int r0 = 2 * ring * (ring - 1); // index of first ring pixel (ring numbering) return r0 + to_ring_local(f, i, j, Nring, 0); } if (ring < 3 * Nside_) { // South polar cap int Nring = 4 * Nside_; int r0 = Ncap_ + (ring - Nside_) * Nring; // index of first ring pixel (ring numbering) int shift = (ring - Nside_) & 1; return r0 + to_ring_local(f, i, j, Nring, shift); } // Equatorial belt int N = 4 * Nside_ - ring; int Nring = 4 * N; int r0 = Npix_ - 2 * N * (N + 1); // index of first ring pixel (ring numbering) return r0 + to_ring_local(f, i, j, Nring, 0); } }; } // namespace HEALPix::HEALPix(const order_type& order) : order_(order) { if (order_ != RING && order_ != NESTED) { throw exception::OrderError("HEALPix: supported ordering: ring, nested", Here()); } } HEALPix::HEALPix(const Spec& spec) : HEALPix(spec.get_string("order", order_default())) {} HEALPix::renumber_type HEALPix::reorder(const order_type& to, size_t Nside) const { ASSERT(to == NESTED || to == RING); auto size = static_cast(grid::reduced::HEALPix::size_from_nside(Nside)); renumber_type ren(size); if (order_ == to) { // no reordering std::iota(ren.begin(), ren.end(), 0); return ren; } auto from_nested = order_ == NESTED; Renumber renumber(Nside); for (int i = 0; i < size; ++i) { ren[i] = from_nested ? renumber.nest_to_ring(i) : renumber.ring_to_nest(i); } return ren; } } // namespace eckit::geo::order eckit-2.0.7/src/eckit/geo/order/Scan.h0000664000175000017500000000303215161702250017610 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Grid.h" #include "eckit/geo/util.h" namespace eckit::spec { class Spec; } namespace eckit::geo::order { class Scan { public: // -- Types using order_type = Grid::order_type; using renumber_type = Grid::renumber_type; using Spec = spec::Spec; // -- Constructors explicit Scan(const order_type& = order_default()); explicit Scan(const Spec&); Scan(const Scan&) = default; Scan(Scan&&) = default; // -- Operators Scan& operator=(const Scan&) = default; Scan& operator=(Scan&&) = default; // -- Methods void order(const order_type& to) { operator=(Scan{to}); } const order_type& order() const { return order_; } renumber_type reorder(const order_type& to, size_t ni, size_t nj) const; renumber_type reorder(const order_type& to, const pl_type&) const; bool is_scan_i_positive() const; bool is_scan_j_positive() const; bool is_scan_alternating() const; // -- Class methods static const order_type& order_default(); private: // -- Members order_type order_; }; } // namespace eckit::geo::order eckit-2.0.7/src/eckit/geo/order/HEALPix.h0000664000175000017500000000223715161702250020124 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Grid.h" namespace eckit::geo::order { class HEALPix { public: // -- Types using order_type = Grid::order_type; using renumber_type = Grid::renumber_type; using Spec = spec::Spec; // -- Constructors explicit HEALPix(const order_type& = order_default()); explicit HEALPix(const Spec&); // -- Methods const order_type& order() const { return order_; } renumber_type reorder(const order_type& to, size_t Nside) const; // -- Class members static const order_type RING; static const order_type NESTED; // -- Class methods static const order_type& order_default() { return RING; } private: // -- Members order_type order_; }; } // namespace eckit::geo::order eckit-2.0.7/src/eckit/geo/order/Scan.cc0000664000175000017500000000624715161702250017761 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to anj jurisdiction. */ #include "eckit/geo/order/Scan.h" #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/spec/Spec.h" namespace eckit::geo::order { static const Scan::order_type IPOS_JPOS{"i+j+"}; static const Scan::order_type IPOS_JNEG{"i+j-"}; static const Scan::order_type INEG_JPOS{"i-j+"}; static const Scan::order_type INEG_JNEG{"i-j-"}; static const Scan::order_type INEGPOS_JPOS{"i-+j+"}; static const Scan::order_type INEGPOS_JNEG{"i-+j-"}; static const Scan::order_type IPOSNEG_JPOS{"i+-j+"}; static const Scan::order_type IPOSNEG_JNEG{"i+-j-"}; static const Scan::order_type JPOS_IPOS{"j+i+"}; static const Scan::order_type JPOS_INEG{"j+i-"}; static const Scan::order_type JNEG_IPOS{"j-i+"}; static const Scan::order_type JNEG_INEG{"j-i-"}; static const Scan::order_type JNEGPOS_IPOS{"j-+i+"}; static const Scan::order_type JNEGPOS_INEG{"j-+i-"}; static const Scan::order_type JPOSNEG_IPOS{"j+-i+"}; static const Scan::order_type JPOSNEG_INEG{"j+-i-"}; static const std::vector MODES{ IPOS_JPOS, IPOS_JNEG, INEG_JPOS, INEG_JNEG, INEGPOS_JPOS, INEGPOS_JNEG, IPOSNEG_JPOS, IPOSNEG_JNEG, JPOS_IPOS, JPOS_INEG, JNEG_IPOS, JNEG_INEG, JNEGPOS_IPOS, JNEGPOS_INEG, JPOSNEG_IPOS, JPOSNEG_INEG, }; Scan::Scan(const order_type& order) : order_(order) { if (std::count(MODES.begin(), MODES.end(), order_) != 1) { throw exception::OrderError("Scan invalid order: '" + order_ + "'", Here()); } } Scan::Scan(const Spec& spec) : Scan(spec.get_string("order", order_default())) {} bool Scan::is_scan_i_positive() const { return order_.find("i+") != order_type::npos; } bool Scan::is_scan_j_positive() const { return order_.find("j+") != order_type::npos; } bool Scan::is_scan_alternating() const { return order_.find("+-") != order_type::npos || order_.find("-+") != order_type::npos; } const Scan::order_type& Scan::order_default() { return IPOS_JNEG; } Scan::renumber_type Scan::reorder(const order_type& to, size_t ni, size_t nj) const { ASSERT(0 < ni && 0 < nj); if (to == order_) { // no reordering renumber_type ren(ni * nj); std::iota(ren.begin(), ren.end(), 0); return ren; } // TODO regular grid reordering NOTIMP; } Scan::renumber_type Scan::reorder(const order_type& to, const pl_type& pl) const { ASSERT(2 <= pl.size()); ASSERT(std::all_of(pl.begin(), pl.end(), [](auto n) { return 2 <= n; })); if (to == order_) { // no reordering auto size = static_cast(std::accumulate(pl.begin(), pl.end(), static_cast(0))); renumber_type ren(size); std::iota(ren.begin(), ren.end(), 0); return ren; } // TODO reduced grid reordering NOTIMP; } } // namespace eckit::geo::order eckit-2.0.7/src/eckit/geo/util.cc0000664000175000017500000000106215161702250016725 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/util.h" namespace eckit::geo::util { template <> pl_type pl_convert(const pl_type& pl) { return pl; } } // namespace eckit::geo::util eckit-2.0.7/src/eckit/geo/PointXYZ.h0000664000175000017500000000715015161702250017322 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit::geo { /** * @brief The PointXYZ class * @details A point on three-dimensional space, in (X, Y, Z) coordinates, linear in space. */ class PointXYZ final : protected std::array { public: // -- Types using container_type = array; using container_type::value_type; // -- Constructors PointXYZ() : PointXYZ(0., 0., 0.) {} PointXYZ(value_type x, value_type y, value_type z) : container_type{x, y, z} {} PointXYZ(const PointXYZ& other) : container_type(other) {} PointXYZ(PointXYZ&& other) : container_type(other) {} // -- Destructor ~PointXYZ() = default; // -- Operators using container_type::operator[]; PointXYZ& operator=(const PointXYZ& other) { container_type::operator=(other); return *this; } PointXYZ& operator=(PointXYZ&& other) { container_type::operator=(other); return *this; } // -- Members value_type X() const { return container_type::operator[](0); } value_type Y() const { return container_type::operator[](1); } value_type Z() const { return container_type::operator[](2); } // -- Methods using container_type::data; static size_t dimensions() { return DIMS; } static value_type distance(const PointXYZ& p, const PointXYZ& q, size_t axis) { return p.distance(q, axis); } static value_type distance(const PointXYZ& p, const PointXYZ& q) { return p.distance(q); } static value_type distance2(const PointXYZ& p, const PointXYZ& q) { return p.distance2(q); } value_type distance(const PointXYZ&, size_t axis) const; value_type distance(const PointXYZ&) const; value_type distance2(const PointXYZ&) const; value_type x(size_t axis) const { return container_type::operator[](axis); } // -- Class members static constexpr size_t DIMS = 3; static constexpr value_type EPS = 1e-9; // -- Friends friend std::ostream& operator<<(std::ostream& out, const PointXYZ& p) { return out << '{' << p.X() << ", " << p.Y() << ", " << p.Z() << '}'; } friend PointXYZ operator-(const PointXYZ& p) { return {-p.X(), -p.Y(), -p.Z()}; } friend PointXYZ operator-(const PointXYZ& p, const PointXYZ& q) { return {p.X() - q.X(), p.Y() - q.Y(), p.Z() - q.Z()}; } friend PointXYZ operator+(const PointXYZ& p, const PointXYZ& q) { return {p.X() + q.X(), p.Y() + q.Y(), p.Z() + q.Z()}; } friend PointXYZ operator*(const PointXYZ& p, value_type d) { return {p.X() * d, p.Y() * d, p.Z() * d}; } friend bool operator==(const PointXYZ& p, const PointXYZ& q) { return p.X() == q.X() && p.Y() == q.Y() && p.Z() == q.Z(); } friend bool operator!=(const PointXYZ& p, const PointXYZ& q) { return !operator==(p, q); } friend value_type dot(const PointXYZ& p, const PointXYZ& q) { return p.X() * q.X() + p.Y() * q.Y() + p.Z() * q.Z(); } friend PointXYZ cross(const PointXYZ& p, const PointXYZ& q) { return {p.Y() * q.Z() - p.Z() * q.Y(), p.Z() * q.X() - p.X() * q.Z(), p.X() * q.Y() - p.Y() * q.X()}; } }; bool points_equal(const PointXYZ&, const PointXYZ&, PointXYZ::value_type eps = PointXYZ::EPS); } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Point.h0000664000175000017500000000157415161702250016713 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/PointLonLat.h" #include "eckit/geo/PointLonLatR.h" #include "eckit/geo/PointXY.h" #include "eckit/geo/PointXYZ.h" namespace eckit::geo { using Point = std::variant; bool points_equal(const Point&, const Point&); bool points_equal(const Point&, const Point&, double eps); std::ostream& operator<<(std::ostream&, const Point&); } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Range.cc0000664000175000017500000000225515161702250017011 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Range.h" #include "eckit/geo/Exceptions.h" namespace eckit::geo { double Range::min() const { auto _a = a(); auto _b = b(); return _a < _b ? _a : _b; } double Range::max() const { auto _a = a(); auto _b = b(); return _a < _b ? _b : _a; } Fraction Range::increment() const { throw exception::GridError("Range::increment not implemented", Here()); } bool Range::periodic() const { throw exception::GridError("Range::periodic not implemented", Here()); } bool Range::includesNorthPole() const { throw exception::GridError("Range::includesNorthPole not implemented", Here()); } bool Range::includesSouthPole() const { throw exception::GridError("Range::includesSouthPole not implemented", Here()); } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Exceptions.h0000664000175000017500000000343115161702250017735 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/exception/Exceptions.h" #include "eckit/spec/Exceptions.h" namespace eckit::geo { class Exception : public eckit::Exception { public: using eckit::Exception::Exception; }; } // namespace eckit::geo namespace eckit::geo::exception { using SeriousBug = ::eckit::SeriousBug; using SpecError = ::eckit::spec::exception::SpecError; class AreaError : public geo::Exception { public: explicit AreaError(const std::string&, const CodeLocation&); }; class FigureError : public geo::Exception { public: explicit FigureError(const std::string&, const CodeLocation&); }; class GridError : public geo::Exception { public: explicit GridError(const std::string&, const CodeLocation&); }; class GridUnknownError : public geo::Exception { public: explicit GridUnknownError(const std::string&, const CodeLocation&); }; class OrderError : public geo::Exception { public: explicit OrderError(const std::string&, const CodeLocation&); }; class ProjectionError : public geo::Exception { public: explicit ProjectionError(const std::string&, const CodeLocation&); }; class RangeError : public geo::Exception { public: explicit RangeError(const std::string&, const CodeLocation&); }; class SearchError : public geo::Exception { public: explicit SearchError(const std::string&, const CodeLocation&); }; } // namespace eckit::geo::exception eckit-2.0.7/src/eckit/geo/container/0000775000175000017500000000000015161702250017424 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/container/PointsContainer.cc0000664000175000017500000000631415161702250023056 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/container/PointsContainer.h" #include "eckit/geo/Exceptions.h" namespace eckit::geo::util { void hash_vector_double(MD5&, const std::vector&); } namespace eckit::geo::container { std::pair, std::vector> PointsContainer::points_to_latlons( const std::vector& points) { std::pair, std::vector> ll; ll.first.reserve(points.size()); ll.second.reserve(points.size()); for (const auto& p : points) { auto q = std::get(p); ll.first.emplace_back(q.lat()); ll.second.emplace_back(q.lon()); } return ll; } std::vector PointsContainer::latlons_to_points(const std::vector& latitudes, const std::vector& longitudes) { ASSERT(longitudes.size() == latitudes.size()); std::vector points; points.reserve(longitudes.size()); for (size_t i = 0; i < longitudes.size(); ++i) { points.emplace_back(PointLonLat{longitudes[i], latitudes[i]}); } return points; } std::pair, std::vector> PointsInstance::to_latlons() const { return PointsContainer::points_to_latlons(points_); } void PointsInstance::hash(MD5&) const { NOTIMP; } PointsLonLatInstance::PointsLonLatInstance(std::vector&& longitudes, std::vector&& latitudes) : longitudes(longitudes), latitudes(latitudes) { ASSERT(longitudes.size() == latitudes.size()); } std::vector PointsLonLatInstance::to_points() const { return latlons_to_points(latitudes, longitudes); } std::pair, std::vector> PointsLonLatInstance::to_latlons() const { return {latitudes, longitudes}; } void PointsLonLatInstance::hash(MD5& hash) const { util::hash_vector_double(hash, latitudes); util::hash_vector_double(hash, longitudes); } PointsLonLatReference::PointsLonLatReference(const std::vector& longitudes, const std::vector& latitudes) : longitudes(longitudes), latitudes(latitudes) { ASSERT(longitudes.size() == latitudes.size()); } std::vector PointsLonLatReference::to_points() const { return latlons_to_points(latitudes, longitudes); } std::pair, std::vector> PointsLonLatReference::to_latlons() const { return {latitudes, longitudes}; } void PointsLonLatReference::hash(MD5& hash) const { util::hash_vector_double(hash, latitudes); util::hash_vector_double(hash, longitudes); } std::pair, std::vector> PointsReference::to_latlons() const { return PointsContainer::points_to_latlons(points_); } void PointsReference::hash(MD5&) const { NOTIMP; } } // namespace eckit::geo::container eckit-2.0.7/src/eckit/geo/container/PointsContainer.h0000664000175000017500000000700415161702250022715 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/Container.h" #include "eckit/geo/Point.h" namespace eckit { class MD5; } namespace eckit::geo::container { class PointsContainer : public Container { protected: PointsContainer() = default; static std::pair, std::vector> points_to_latlons(const std::vector&); static std::vector latlons_to_points(const std::vector& latitudes, const std::vector& longitudes); public: [[nodiscard]] virtual std::vector to_points() const = 0; [[nodiscard]] virtual std::pair, std::vector> to_latlons() const = 0; virtual void hash(MD5&) const = 0; }; struct PointsInstance : PointsContainer { explicit PointsInstance(std::vector&& points) : points_(points) {} Point get(size_t index) const override { return points_.at(index); } size_t size() const override { return points_.size(); } [[nodiscard]] std::vector to_points() const override { return points_; } [[nodiscard]] std::pair, std::vector> to_latlons() const override; void hash(MD5&) const override; private: const std::vector points_; }; struct PointsLonLatInstance : PointsContainer { explicit PointsLonLatInstance(std::vector&& longitudes, std::vector&& latitudes); Point get(size_t index) const override { return PointLonLat{longitudes.at(index), latitudes.at(index)}; } size_t size() const override { return longitudes.size(); } [[nodiscard]] std::vector to_points() const override; [[nodiscard]] std::pair, std::vector> to_latlons() const override; void hash(MD5&) const override; private: const std::vector longitudes; const std::vector latitudes; }; struct PointsLonLatReference : PointsContainer { explicit PointsLonLatReference(const std::vector& longitudes, const std::vector& latitudes); Point get(size_t index) const override { return PointLonLat{longitudes.at(index), latitudes.at(index)}; } size_t size() const override { return longitudes.size(); } [[nodiscard]] std::vector to_points() const override; [[nodiscard]] std::pair, std::vector> to_latlons() const override; void hash(MD5&) const override; private: const std::vector& longitudes; const std::vector& latitudes; }; struct PointsReference : PointsContainer { explicit PointsReference(const std::vector& points) : points_(points) {} Point get(size_t index) const override { return points_.at(index); } size_t size() const override { return points_.size(); } [[nodiscard]] std::vector to_points() const override { return points_; } [[nodiscard]] std::pair, std::vector> to_latlons() const override; void hash(MD5&) const override; private: const std::vector& points_; }; } // namespace eckit::geo::container eckit-2.0.7/src/eckit/geo/GreatCircle.h0000664000175000017500000000316515161702250020004 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/PointLonLat.h" namespace eckit::geo { class GreatCircle { public: /// Great circle given two points in geographic coordinates GreatCircle(const PointLonLat&, const PointLonLat&); /// Great circle latitude given longitude, see http://www.edwilliams.org/avform.htm#Int std::vector latitude(double lon) const; /// Great circle longitude given latitude, see http://www.edwilliams.org/avform.htm#Par std::vector longitude(double lat) const; /// If great circle crosses the poles (meridian/anti-meridian) bool crossesPoles() const; /** * @brief Calculate great circle course between two points * * @details Calculates the direction (clockwise from North) of a great circle arc between two points. Returns the * direction (angle) of the arc at each, normalised to the range of atan2 (usually (-180, 180]). All input and * output values are in units of degrees. * * @ref https://en.wikipedia.org/wiki/Great-circle_navigation */ std::pair course() const; private: const PointLonLat A_; const PointLonLat B_; bool crossesPoles_; }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Projection.cc0000664000175000017500000001036215161702250020067 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Projection.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/Figure.h" #include "eckit/geo/eckit_geo_config.h" #include "eckit/geo/figure/Earth.h" #include "eckit/geo/projection/None.h" #include "eckit/geo/share/Projection.h" #include "eckit/geo/util/mutex.h" #include "eckit/parser/YAMLParser.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Layered.h" #include "eckit/types/FloatCompare.h" #if eckit_HAVE_PROJ #include "eckit/geo/projection/PROJ.h" #endif namespace eckit::geo { namespace { util::recursive_mutex MUTEX; class lock_type { util::lock_guard lock_guard_{MUTEX}; }; } // namespace Projection::Projection(Figure* figure_ptr) : figure_(figure_ptr != nullptr ? figure_ptr : new figure::Earth) { ASSERT(figure_); } const Figure& Projection::figure() const { ASSERT(figure_); return *figure_; } const Projection::Spec& Projection::spec() const { if (!spec_) { spec_ = std::make_shared(); ASSERT(spec_); auto& custom = *spec_; fill_spec(custom); if (std::string name; !custom.empty() && ProjectionSpecByName::instance().match(custom, name)) { custom.clear(); custom.set(className(), name); } } return *spec_; } std::string Projection::proj_str() const { #if eckit_HAVE_PROJ return projection::PROJ::proj_str(dynamic_cast(spec())); #else NOTIMP; #endif } const Projection& Projection::projection_default() { static const projection::None proj; return proj; } Projection* Projection::make_from_spec(const Spec& spec) { if (std::string type; spec.get("type", type)) { return ProjectionFactoryType::instance().get(type).create(spec); } throw exception::SpecError("Projection: cannot build grid without 'type'", Here()); } void Projection::fill_spec(spec::Custom& custom) const { figure_->fill_spec(custom); if (!types::is_approximately_equal(false_.X(), 0.)) { custom.set("x_0", false_.X()); } if (!types::is_approximately_equal(false_.Y(), 0.)) { custom.set("y_0", false_.Y()); } } const Projection* ProjectionFactory::make_from_string(const std::string& str) { std::unique_ptr spec(spec::Custom::make_from_value(YAMLParser::decodeString(str))); return instance().make_from_spec_(*spec); } ProjectionFactory& ProjectionFactory::instance() { static ProjectionFactory obj; return obj; } const Projection* ProjectionFactory::make_from_spec_(const Projection::Spec& spec) const { lock_type lock; std::unique_ptr cfg(make_spec_(spec)); if (std::string type; cfg->get("type", type)) { return ProjectionFactoryType::instance().get(type).create(*cfg); } list(Log::error() << "Projection: cannot build projection without 'type', choices are: "); throw exception::SpecError("Projection: cannot build projection without 'type'", Here()); } Projection::Spec* ProjectionFactory::make_spec_(const Projection::Spec& spec) const { lock_type lock; share::Projection::instance(); auto* cfg = new spec::Layered(spec); ASSERT(cfg != nullptr); // hardcoded, interpreted options (contributing to projectionspec) if (spec.has("proj")) { cfg->push_back(new spec::Custom{{"type", "proj"}}); } else if (spec.has("rotation")) { std::vector rotation; spec.get("rotation", rotation); cfg->push_back(new spec::Custom{{"type", "rotation"}, {"rotation", rotation}}); } return cfg; } std::ostream& ProjectionFactory::list_(std::ostream& out) const { lock_type lock; share::Projection::instance(); out << ProjectionSpecByName::instance() << std::endl; out << ProjectionFactoryType::instance() << std::endl; return out; } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Figure.cc0000664000175000017500000000700615161702250017175 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Figure.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/figure/Earth.h" #include "eckit/geo/figure/OblateSpheroid.h" #include "eckit/geo/figure/Sphere.h" #include "eckit/geo/util/mutex.h" #include "eckit/parser/YAMLParser.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo { static util::recursive_mutex MUTEX; class lock_type { util::lock_guard lock_guard_{MUTEX}; }; double Figure::R() const { NOTIMP; } double Figure::a() const { NOTIMP; } double Figure::b() const { NOTIMP; } double Figure::area() const { NOTIMP; } double Figure::area(const area::BoundingBox&) const { NOTIMP; } spec::Custom* Figure::spec() const { auto* custom = new spec::Custom; ASSERT(custom != nullptr); fill_spec(*custom); return custom; } std::string Figure::spec_str() const { std::unique_ptr custom(spec()); return custom->str(); } bool Figure::spherical() const { return types::is_approximately_equal(a(), b()); } double Figure::eccentricity() const { return figure::OblateSpheroid::eccentricity(a(), b()); } double Figure::flattening() const { return figure::OblateSpheroid::flattening(a(), b()); } void Figure::fill_spec(spec::Custom& custom) const { static const std::map, std::string> KNOWN{ {std::shared_ptr
{new figure::Grs80}, "grs80"}, {std::shared_ptr
{new figure::Wgs84}, "wgs84"}, }; for (const auto& [figure, name] : KNOWN) { if (types::is_approximately_equal(figure->a(), a()) && types::is_approximately_equal(figure->b(), b())) { custom.set("figure", name); return; } } if (types::is_approximately_equal(a(), b())) { custom.set("R", R()); } else { custom.set("a", a()); custom.set("b", b()); } } FigureFactory& FigureFactory::instance() { static FigureFactory obj; return obj; } Figure* FigureFactory::make_from_string(const std::string& str) { std::unique_ptr spec(spec::Custom::make_from_value(YAMLParser::decodeString(str))); return instance().make_from_spec_(*spec); } Figure* FigureFactory::make_from_spec_(const Figure::Spec& spec) const { lock_type lock; if (std::string figure; spec.get("figure", figure)) { return Factory
::instance().get(figure).create(); } if (double a = 0., b = 0.; (spec.get("a", a) && spec.get("b", b)) || (spec.get("semi_major_axis", a) && spec.get("semi_minor_axis", b))) { return types::is_approximately_equal(a, b) ? static_cast(new figure::Sphere(a)) : new figure::OblateSpheroid(a, b); } if (double R = 0.; spec.get("R", R) || spec.get("radius", R)) { return new figure::Sphere(R); } const auto* msg = "Figure: cannot build figure without 'R'/'a'/'b' or 'radius'/'semi_major_axis'/'semi_minor_axis'"; Log::error() << msg << std::endl; throw exception::SpecError(msg, Here()); } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/figure/0000775000175000017500000000000015161702250016723 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/figure/Sphere.h0000664000175000017500000000517515161702250020332 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Figure.h" namespace eckit::geo::figure { class Sphere : public Figure { public: // -- Constructors explicit Sphere(double R); explicit Sphere(const Spec&); // -- Overridden methods double R() const override { return R_; } double a() const override { return R_; } double b() const override { return R_; } /// Surface area [L^2] double area() const override { return area(R_); } /// Surface area between parallels and meridians [L^2] double area(const area::BoundingBox& bbox) const override { return area(R_, bbox); } // -- Class methods /// Great-circle central angle between two points [radian] static double centralAngle(const PointLonLat&, const PointLonLat&); /// Great-circle central angle between two points (Cartesian coordinates) static double centralAngle(double radius, const PointXYZ&, const PointXYZ&); /// Great-circle distance between two points static double distance(double radius, const PointLonLat&, const PointLonLat&); /// Great-circle distance between two points (Cartesian coordinates) static double distance(double radius, const PointXYZ&, const PointXYZ&); /// Surface area [L^2] static double area(double radius); /// Surface area between parallels and meridians [L^2] static double area(double radius, const area::BoundingBox&); /// Great-circle intermediate latitude provided two circle points and intermediate longitude [degree] static double greatCircleLatitudeGivenLongitude(const PointLonLat&, const PointLonLat&, double lon); /// Great-circle intermediate longitude(s) provided two circle points and intermediate latitude [degree] static void greatCircleLongitudeGivenLatitude(const PointLonLat&, const PointLonLat&, double lat, double& lon1, double& lon2); /// Convert spherical to Cartesian coordinates static PointXYZ convertSphericalToCartesian(double radius, const PointLonLat&, double height = 0.); /// Convert Cartesian to spherical coordinates static PointLonLat convertCartesianToSpherical(double radius, const PointXYZ&); private: // -- Members const double R_; }; } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/figure/OblateSpheroid.h0000664000175000017500000000331015161702250021775 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Figure.h" namespace eckit::geo::figure { class OblateSpheroid : public Figure { public: // -- Constructors OblateSpheroid(double a, double b); explicit OblateSpheroid(const Spec&); // -- Overridden methods double R() const override; double a() const override { return a_; } double b() const override { return b_; } /// Surface area [L^2] double area() const override { return _area(a_, b_); } /// Surface area between parallels and meridians [L^2] double area(const area::BoundingBox& bbox) const override { return _area(a_, b_, bbox); } // -- Class methods /// Radius static double R(double a, double b); /// Elliptic eccentricity static double eccentricity(double a, double b); /// Flattening static double flattening(double a, double b); /// Surface area [L^2] static double _area(double a, double b); /// Surface area between parallels and meridians [L^2] static double _area(double a, double b, const area::BoundingBox&); /// Convert geocentric coordinates to Cartesian static PointXYZ convertSphericalToCartesian(double a, double b, const PointLonLat&, double height = 0.); private: // -- Members const double a_; const double b_; }; } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/figure/OblateSpheroidT.h0000664000175000017500000000232215161702250022123 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/figure/OblateSpheroid.h" namespace eckit::geo::figure { /// Oblate spheroid parametrised with a geodetic datum template class OblateSpheroidT : public Figure { public: // -- Constructors OblateSpheroidT() = default; // -- Overridden methods double R() const override { return OblateSpheroid::R(DATUM::a, DATUM::b); } double a() const override { return DATUM::a; } double b() const override { return DATUM::b; } /// Surface area [L^2] double area() const override { return OblateSpheroid::_area(DATUM::a, DATUM::b); } /// Surface area between parallels and meridians [L^2] double area(const area::BoundingBox& bbox) const override { return OblateSpheroid::_area(DATUM::a, DATUM::b, bbox); } }; } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/figure/Earth.cc0000664000175000017500000000132615161702250020277 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/figure/Earth.h" namespace eckit::geo::figure { static const FigureBuilder REGISTER1("earth"); static const FigureBuilder REGISTER2("grs80"); static const FigureBuilder REGISTER3("wgs84"); const Earth EARTH; const Grs80 GRS80; const Wgs84 WGS84; } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/figure/SphereT.h0000664000175000017500000000535115161702250020452 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/figure/Sphere.h" namespace eckit::geo::area { class BoundingBox; } namespace eckit::geo::figure { /// Sphere parametrised with a geodetic datum template class SphereT : public Figure { public: // -- Constructors SphereT() = default; // -- Overridden methods double R() const override { return DATUM::radius; } double a() const override { return DATUM::radius; } double b() const override { return DATUM::radius; } /// Surface area [L^2] double area() const override { return _area(); } /// Surface area between parallels and meridians [L^2] double area(const area::BoundingBox& bbox) const override { return _area(bbox); } // -- Class methods /// Sphere radius inline static double radius() { return DATUM::radius; } /// Great-circle central angle between two points [radian] inline static double centralAngle(const PointLonLat& A, const PointLonLat& B) { return Sphere::centralAngle(A, B); } /// Great-circle central angle between two points (Cartesian coordinates) in radians inline static double centralAngle(const PointXYZ& A, const PointXYZ& B) { return Sphere::centralAngle(DATUM::radius, A, B); } /// Great-circle distance between two points inline static double distance(const PointLonLat& A, const PointLonLat& B) { return Sphere::distance(DATUM::radius, A, B); } /// Great-circle distance between two points (Cartesian coordinates) inline static double distance(const PointXYZ& A, const PointXYZ& B) { return Sphere::distance(DATUM::radius, A, B); } /// Surface area [L^2] inline static double _area() { return Sphere::area(DATUM::radius); } /// Surface area between parallels and meridians [L^2] inline static double _area(const area::BoundingBox& bbox) { return Sphere::area(DATUM::radius, bbox); } /// Convert spherical to Cartesian coordinates inline static PointXYZ _convertSphericalToCartesian(const PointLonLat& P, double height = 0.) { return Sphere::convertSphericalToCartesian(DATUM::radius, P, height); } /// Convert Cartesian to spherical coordinates inline static PointLonLat _convertCartesianToSpherical(const PointXYZ& P) { return Sphere::convertCartesianToSpherical(DATUM::radius, P); } }; } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/figure/Sphere.cc0000664000175000017500000001410215161702250020456 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/figure/Sphere.h" #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/GreatCircle.h" #include "eckit/geo/PointLonLat.h" #include "eckit/geo/PointXYZ.h" #include "eckit/geo/area/BoundingBox.h" #include "eckit/geo/util.h" #include "eckit/spec/Spec.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::figure { Sphere::Sphere(double R) : R_(R) { if (!(0. < R_)) { throw BadValue("Sphere::R requires 0 < R", Here()); } } Sphere::Sphere(const Spec& spec) : Sphere(spec.get_double("R")) {} double Sphere::centralAngle(const PointLonLat& P1, const PointLonLat& P2) { /* * Δσ = atan( ((cos(ϕ2) * sin(Δλ))^2 + (cos(ϕ1) * sin(ϕ2) - sin(ϕ1) * cos(ϕ2) * cos(Δλ))^2) / * (sin(ϕ1) * sin(ϕ2) + cos(ϕ1) * cos(ϕ2) * cos(Δλ)) ) * * @article{doi:10.1179/sre.1975.23.176.88, * author = {T. Vincenty}, * title = {Direct and Inverse Solutions of Geodesics on the Ellipsoid With Application of Nested Equations}, * journal = {Survey Review}, * volume = {23}, * number = {176}, * pages = {88-93}, * year = {1975}, * doi = {10.1179/sre.1975.23.176.88} * } */ const auto Q1 = PointLonLat::make(P1.lon(), P1.lat(), -180.); const auto Q2 = PointLonLat::make(P2.lon(), P2.lat(), -180.); const auto phi1 = util::DEGREE_TO_RADIAN * Q1.lat(); const auto phi2 = util::DEGREE_TO_RADIAN * Q2.lat(); const auto lambda = util::DEGREE_TO_RADIAN * PointLonLat::normalise_angle_to_minimum(Q1.lon() - Q2.lon(), -180.); const auto cp1 = std::cos(phi1); const auto sp1 = std::sin(phi1); const auto cp2 = std::cos(phi2); const auto sp2 = std::sin(phi2); const auto cl = std::cos(lambda); const auto sl = std::sin(lambda); auto squared = [](double x) { return x * x; }; const auto angle = std::atan2(std::sqrt(squared(cp2 * sl) + squared(cp1 * sp2 - sp1 * cp2 * cl)), sp1 * sp2 + cp1 * cp2 * cl); if (types::is_approximately_equal(angle, 0.)) { return 0.; } ASSERT(angle > 0.); return angle; } double Sphere::centralAngle(double radius, const PointXYZ& P1, const PointXYZ& P2) { ASSERT(radius > 0.); // Δσ = 2 * asin( chord / 2 ) const auto d2 = PointXYZ::distance2(P1, P2); if (types::is_approximately_equal(d2, 0.)) { return 0.; } const auto chord = std::sqrt(d2) / radius; const auto angle = std::asin(chord * 0.5) * 2.; return angle; } double Sphere::distance(double radius, const PointLonLat& P1, const PointLonLat& P2) { return radius * centralAngle(P1, P2); } double Sphere::distance(double radius, const PointXYZ& P1, const PointXYZ& P2) { return radius * centralAngle(radius, P1, P2); } double Sphere::area(double radius) { ASSERT(radius > 0.); return 4. * M_PI * radius * radius; } double Sphere::area(double radius, const area::BoundingBox& bbox) { ASSERT(radius > 0.); // Set longitude and latitude fractions auto lonf = bbox.periodic() ? 1. : (bbox.east() - bbox.west()) / PointLonLat::FULL_ANGLE; ASSERT(0. <= lonf && lonf <= 1.); auto sn = std::sin(util::DEGREE_TO_RADIAN * bbox.north()); auto ss = std::sin(util::DEGREE_TO_RADIAN * bbox.south()); auto latf = 0.5 * (sn - ss); // Calculate area return area(radius) * latf * lonf; } double Sphere::greatCircleLatitudeGivenLongitude(const PointLonLat& P1, const PointLonLat& P2, double Clon) { GreatCircle gc(P1, P2); auto lat = gc.latitude(Clon); return lat.size() == 1 ? lat[0] : std::numeric_limits::signaling_NaN(); } void Sphere::greatCircleLongitudeGivenLatitude(const PointLonLat& P1, const PointLonLat& P2, double Clat, double& Clon1, double& Clon2) { GreatCircle gc(P1, P2); auto lon = gc.longitude(Clat); Clon1 = lon.size() > 0 ? lon[0] : std::numeric_limits::signaling_NaN(); Clon2 = lon.size() > 1 ? lon[1] : std::numeric_limits::signaling_NaN(); } PointXYZ Sphere::convertSphericalToCartesian(double radius, const PointLonLat& P, double height) { ASSERT(radius > 0.); /* * See https://en.wikipedia.org/wiki/Reference_ellipsoid#Coordinates * numerical conditioning for both ϕ (poles) and λ (Greenwich/Date Line). * * cos φ = sqrt( 1 - sin^2 φ) is better conditioned than explicit cos φ, and * coupled with λ in [-180°, 180°[ the accuracy of the trigonometric * functions is the same (before converting/multiplying its angle argument * to radian) and explicitly chosing -180° over 180° for longitude. * * These conditionings combined project accurately to sphere poles and quadrants. */ const auto Q = PointLonLat::make(P.lon(), P.lat(), -180.); const auto lambda = util::DEGREE_TO_RADIAN * Q.lon(); const auto phi = util::DEGREE_TO_RADIAN * Q.lat(); const auto sp = std::sin(phi); const auto cp = std::sqrt(1. - sp * sp); const auto sl = std::abs(Q.lon()) < 180. ? std::sin(lambda) : 0.; const auto cl = std::abs(Q.lon()) > 90. ? std::cos(lambda) : std::sqrt(1. - sl * sl); return {(radius + height) * cp * cl, (radius + height) * cp * sl, (radius + height) * sp}; } PointLonLat Sphere::convertCartesianToSpherical(double radius, const PointXYZ& A) { ASSERT(radius > 0.); // numerical conditioning for both z (poles) and y const auto x = A[0]; const auto y = types::is_approximately_equal(A[1], 0.) ? 0. : A[1]; const auto z = std::min(radius, std::max(-radius, A[2])) / radius; return {util::RADIAN_TO_DEGREE * std::atan2(y, x), util::RADIAN_TO_DEGREE * std::asin(z)}; } } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/figure/OblateSpheroid.cc0000664000175000017500000000651415161702250022144 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/figure/OblateSpheroid.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/PointLonLat.h" #include "eckit/geo/PointXYZ.h" #include "eckit/geo/area/BoundingBox.h" #include "eckit/geo/util.h" #include "eckit/spec/Spec.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::figure { OblateSpheroid::OblateSpheroid(double a, double b) : a_(a), b_(b) { if (!(0. < b && b_ <= a_)) { throw BadValue("OblateSpheroid::R requires 0 < b <= a", Here()); } } OblateSpheroid::OblateSpheroid(const Spec& spec) : OblateSpheroid(spec.get_double("a"), spec.get_double("b")) {} double OblateSpheroid::R() const { return R(a_, b_); } double OblateSpheroid::R(double a, double b) { if (types::is_approximately_equal(a, b)) { return a; } throw BadValue("OblateSpheroid::R requires a ~= b", Here()); } double OblateSpheroid::eccentricity(double a, double b) { ASSERT(0. < b && b <= a); return std::sqrt(1. - b * b / (a * a)); } double OblateSpheroid::flattening(double a, double b) { ASSERT(0. < b && b <= a); return (a - b) / a; } double OblateSpheroid::_area(double a, double b) { const auto e = eccentricity(a, b); const auto A = 2. * M_PI * a * a * (1. + (1. - e * e) * std::atanh(e) / e); return A; } double OblateSpheroid::_area(double a, double b, const area::BoundingBox& bbox) { auto f = [](double phi, double e) { auto sin_phi = std::sin(phi); if (types::is_approximately_equal(sin_phi, 0.)) { return sin_phi; } auto denom = 1. - e * e * sin_phi * sin_phi; return (sin_phi / denom + (0.5 / e) * std::log((1. + e * sin_phi) / (1. - e * sin_phi))); }; const auto phi1 = util::DEGREE_TO_RADIAN * bbox.south(); const auto phi2 = util::DEGREE_TO_RADIAN * bbox.north(); const auto dlam = util::DEGREE_TO_RADIAN * (bbox.east() - bbox.west()); const auto e = eccentricity(a, b); const auto A = dlam * a * b * (f(phi2, e) - f(phi1, e)); return A; } PointXYZ OblateSpheroid::convertSphericalToCartesian(double a, double b, const PointLonLat& P, double height) { ASSERT(0. < b && 0. < a); // See https://en.wikipedia.org/wiki/Reference_ellipsoid#Coordinates // numerical conditioning for both ϕ (poles) and λ (Greenwich/Date Line) const auto Q = PointLonLat::make(P.lon(), P.lat(), -180.); const auto lambda = util::DEGREE_TO_RADIAN * Q.lon(); const auto phi = util::DEGREE_TO_RADIAN * Q.lat(); const auto sp = std::sin(phi); const auto cp = std::sqrt(1. - sp * sp); const auto sl = std::abs(Q.lon()) < 180. ? std::sin(lambda) : 0.; const auto cl = std::abs(Q.lon()) > 90. ? std::cos(lambda) : std::sqrt(1. - sl * sl); const auto N_phi = a * a / std::sqrt(a * a * cp * cp + b * b * sp * sp); return {(N_phi + height) * cp * cl, (N_phi + height) * cp * sl, (N_phi * (b * b) / (a * a) + height) * sp}; } } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/figure/Earth.h0000664000175000017500000000242415161702250020141 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/figure/OblateSpheroidT.h" #include "eckit/geo/figure/SphereT.h" namespace eckit::geo::figure { struct DatumIFS { static constexpr double radius = 6371229.; static constexpr double a = radius; static constexpr double b = radius; }; struct DatumGRIB1 { static constexpr double radius = 6367470.; static constexpr double a = radius; static constexpr double b = radius; }; struct DatumGrs80 { static constexpr double a = 6378137.; static constexpr double b = 6356752.314140; }; struct DatumWgs84 { static constexpr double a = 6378137.; static constexpr double b = 6356752.314245; }; using Earth = SphereT; using Grs80 = OblateSpheroidT; using Wgs84 = OblateSpheroidT; extern const Earth EARTH; extern const Grs80 GRS80; extern const Wgs84 WGS84; } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/figure/UnitSphere.cc0000664000175000017500000000107615161702250021324 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/figure/UnitSphere.h" namespace eckit::geo::figure { static const FigureBuilder REGISTER("unit-sphere"); } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/figure/UnitSphere.h0000664000175000017500000000137615161702250021171 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/figure/SphereT.h" namespace eckit::geo::figure { /// Definition of a unit datum struct DatumUnit { static constexpr double radius = 1.; static constexpr double a = radius; static constexpr double b = radius; }; /// Definition of a unit sphere using UnitSphere = SphereT; } // namespace eckit::geo::figure eckit-2.0.7/src/eckit/geo/Search.cc0000664000175000017500000000601215161702250017155 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Search.h" #include "eckit/config/Resource.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/Grid.h" #include "eckit/geo/Iterator.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/geo/Trace.h" #include "eckit/geo/projection/LonLatToXYZ.h" #include "eckit/log/ResourceUsage.h" #include "eckit/thread/AutoLock.h" namespace eckit::geo { static std::string extract_loader(const spec::Spec& spec) { static const std::string default_loader = spec.get_bool("caching", LibEcKitGeo::caching()) ? std::string{LibResource("$ECKIT_GEO_SEARCH_LOADER;ecKitGeoSearchLoader", "mapped-cache-file")} : "memory"; return spec.get_string("eckit-geo-search-trees", default_loader); } Search::Search(const Grid& r, const spec::Spec& spec) { ResourceUsage usage("Search: build tree"); tree_.reset(search::TreeFactory::build(extract_loader(spec), r)); AutoLock lock(*tree_); if (!tree_->ready()) { build(r); tree_->commit(); } } Search::PointValueType Search::closestPoint(const Search::PointType& pt) const { return tree_->nearestNeighbour(pt); } void Search::closestNPoints(const PointType& pt, size_t n, std::vector& closest) const { // Small optimisation if (n == 1) { closest.clear(); closest.push_back(closestPoint(pt)); return; } closest = tree_->kNearestNeighbours(pt, n); } void Search::closestWithinRadius(const PointType& pt, double radius, std::vector& closest) const { closest = tree_->findInSphere(pt, radius); } void Search::build(const Grid& r) { Trace trace("eckit::geo::Search: building k-d tree"); const auto npts = r.size(); ASSERT(npts == tree_->itemCount()); // assumes grid describes lat/lon projection::LonLatToXYZ to_xyz; size_t index = 0; if (static bool fast = Resource("$ECKIT_GEO_SEARCH_FAST_BUILD", true); fast) { std::vector points; points.reserve(npts); for (const auto& p : r) { auto q = to_xyz.fwd(std::get(p)); points.emplace_back(Search::PointType{q.X(), q.Y(), q.Z()}, index++); } tree_->build(points); return; } for (const auto& p : r) { auto q = to_xyz.fwd(std::get(p)); tree_->insert({Search::PointType{q.X(), q.Y(), q.Z()}, index++}); } } void Search::print(std::ostream& out) const { tree_->statsPrint(out, false); tree_->statsReset(); } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/Area.h0000664000175000017500000000614715161702250016473 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/geo/Point.h" #include "eckit/memory/Builder.h" #include "eckit/memory/Factory.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Generator.h" namespace eckit::geo::area { class BoundingBox; } namespace eckit::geo { class Area { public: // -- Types using builder_t = BuilderT1; using Spec = spec::Spec; using ARG1 = const Spec&; // -- Constructors Area() noexcept = default; Area(const Area&) = default; Area(Area&&) = default; // -- Destructor virtual ~Area() = default; // -- Operators Area& operator=(const Area&) = default; Area& operator=(Area&&) = default; // -- Methods [[nodiscard]] const Spec& spec() const; std::string spec_str() const { return spec().str(); } virtual void fill_spec(spec::Custom&) const = 0; virtual const std::string& type() const = 0; virtual bool intersects(area::BoundingBox&) const; virtual bool contains(const Point&) const; virtual double area() const; // -- Class methods static std::string className() { return "area"; } static const Area& area_default(); private: // -- Members mutable std::shared_ptr spec_; // -- Friends friend class Grid; // -- Friends friend bool operator==(const Area& a, const Area& b) { return a.spec_str() == b.spec_str(); } friend bool operator!=(const Area& a, const Area& b) { return !(a == b); } }; using AreaFactoryType = Factory; using AreaSpecByName = spec::GeneratorT>; template using AreaRegisterType = ConcreteBuilderT1; template using AreaRegisterName = spec::ConcreteSpecGeneratorT1; struct AreaFactory { [[nodiscard]] static const Area* build(const Area::Spec& spec) { return instance().make_from_spec_(spec); } [[nodiscard]] static const Area* make_from_string(const std::string&); [[nodiscard]] static Area::Spec* make_spec(const Area::Spec& spec) { return instance().make_spec_(spec); } static void add_library(const std::string& lib, Area::Spec* spec) { return instance().add_library_(lib, spec); } static std::ostream& list(std::ostream& out) { return instance().list_(out); } private: static AreaFactory& instance(); [[nodiscard]] const Area* make_from_spec_(const Area::Spec&) const; [[nodiscard]] Area::Spec* make_spec_(const Area::Spec&) const; void add_library_(const std::string& lib, Area::Spec* spec); std::ostream& list_(std::ostream&) const; std::map> libraries_; }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/grid/0000775000175000017500000000000015161702250016367 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/grid/Unstructured.cc0000664000175000017500000000466515161702250021420 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/Unstructured.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/container/PointsContainer.h" #include "eckit/geo/iterator/Unstructured.h" #include "eckit/spec/Custom.h" namespace eckit::geo::grid { Grid::iterator Unstructured::cbegin() const { return iterator{new geo::iterator::Unstructured(*this, 0, container_)}; } Grid::iterator Unstructured::cend() const { return iterator{new geo::iterator::Unstructured(*this)}; } Unstructured::Unstructured(const Spec& spec) : Unstructured(new container::PointsLonLatInstance{spec.get_double_vector("longitudes"), spec.get_double_vector("latitudes")}) {} Unstructured::Unstructured(const std::vector& longitudes, const std::vector& latitudes) : Unstructured(new container::PointsLonLatReference{longitudes, latitudes}) {} Unstructured::Unstructured(const std::vector& points) : Unstructured(new container::PointsReference{points}) {} Unstructured::Unstructured(std::vector&& points) : Unstructured(new container::PointsInstance{std::move(points)}) {} Unstructured::Unstructured(container::PointsContainer* container) : container_{container} {} size_t Unstructured::size() const { return container_->size(); } Grid::BoundingBox* Unstructured::calculate_bbox() const { return new BoundingBox; } std::vector Unstructured::to_points() const { return container_->to_points(); } std::pair, std::vector > Unstructured::to_latlons() const { return container_->to_latlons(); } Grid::Spec* Unstructured::spec(const std::string& uid) { if (!GridSpecByUID::instance().exists(uid)) { throw exception::GridUnknownError("Unstructured: unknown grid uid '" + uid + "'", Here()); } return GridSpecByUID::instance().get(uid).spec(); } void Unstructured::fill_spec(spec::Custom& custom) const { Grid::fill_spec(custom); custom.set("type", "unstructured"); custom.set("uid", uid()); } } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/grid/Unstructured.h0000664000175000017500000000425515161702250021255 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/Grid.h" #include "eckit/geo/container/PointsContainer.h" namespace eckit::geo::iterator { class Unstructured; } namespace eckit::geo::grid { class Unstructured : public Grid { public: // -- Constructors explicit Unstructured(const Spec&); explicit Unstructured(const std::vector& longitudes, const std::vector& latitudes); explicit Unstructured(const std::vector&); explicit Unstructured(std::vector&&); // -- Methods std::shared_ptr container() const { return container_; } // -- Overridden methods iterator cbegin() const override; iterator cend() const override; size_t size() const override; std::vector shape() const override { return {size()}; } BoundingBox* calculate_bbox() const override; bool includesNorthPole() const override { return true; } bool includesSouthPole() const override { return true; } bool isPeriodicWestEast() const override { return true; } [[nodiscard]] std::vector to_points() const override; [[nodiscard]] std::pair, std::vector> to_latlons() const override; // -- Class methods [[nodiscard]] static Spec* spec(const std::string& uid); [[nodiscard]] static Unstructured* make_from_latlon(const std::vector&, const std::vector&); protected: // -- Constructors explicit Unstructured(container::PointsContainer*); private: // -- Members std::shared_ptr container_; // -- Overridden methods void fill_spec(spec::Custom&) const override; // -- Friends friend class geo::iterator::Unstructured; }; } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/grid/unstructured/0000775000175000017500000000000015161702250021136 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/grid/unstructured/ICON.cc0000664000175000017500000001134715161702250022203 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/unstructured/ICON.h" #include #include #include #include "eckit/codec/codec.h" #include "eckit/filesystem/PathName.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/geo/cache/Record.h" #include "eckit/geo/container/PointsContainer.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Spec.h" namespace eckit::geo::grid::unstructured { static const ICON::ICONRecord& icon_record(const spec::Spec& spec) { static cache::Record cache(LibEcKitGeo::cacheDir() + "/grid/icon"); return cache.get(spec); } ICON::ICON(const ICONRecord& record, const uid_type& uid, const std::string& arrangement, const std::string& name) : Unstructured(new container::PointsLonLatReference{record.longitudes_, record.latitudes_}), name_(name), arrangement_(arrangement_from_string(arrangement)), record_(record) {} ICON::ICON(const Spec& spec) : ICON(icon_record(spec), spec.get_string("icon_uid"), spec.get_string("icon_arrangement"), spec.get_string("name")) { ASSERT(container()); reset_uid(spec.get_string("icon_uid")); } ICON::ICON(uid_type uid) : ICON(*std::unique_ptr(GridFactory::make_spec(spec::Custom({{"uid", uid}})))) {} ICON::ICON(const std::string& name, Arrangement a) : ICON(*std::unique_ptr( GridFactory::make_spec(spec::Custom({{"grid", name + '_' + arrangement_to_string(a)}})))) {} std::string ICON::arrangement() const { return arrangement_to_string(arrangement_); } ICON::ICONRecord::bytes_t ICON::ICONRecord::footprint() const { return sizeof(longitudes_.front()) * longitudes_.size() + sizeof(latitudes_.front()) * latitudes_.size(); } size_t ICON::ICONRecord::n() const { return latitudes_.size(); } void ICON::ICONRecord::read(const PathName& p) { codec::RecordReader reader(p); int32_t version = 0; reader.read("version", version).wait(); if (version == 0) { reader.read("latitude", latitudes_); reader.read("longitude", longitudes_); reader.wait(); ASSERT(latitudes_.size() == longitudes_.size()); return; } throw SeriousBug("ICON: unsupported version", Here()); } void ICON::ICONRecord::check(const Spec& spec) const { auto _n = static_cast(n()); ASSERT(_n > 0); if (std::vector shape; spec.get("shape", shape)) { ASSERT(shape.size() == 1 && shape.front() == _n); } ASSERT(_n == longitudes_.size()); ASSERT(_n == latitudes_.size()); } Grid::Spec* ICON::spec(const std::string& name) { return GridSpecByUID::instance().get(name).spec(); } Arrangement ICON::arrangement_from_string(const std::string& str) { return str == "C" ? Arrangement::ICON_C : str == "V" ? Arrangement::ICON_V : str == "E" ? Arrangement::ICON_E : throw SeriousBug("ICON: unsupported arrangement '" + str + "'"); } std::string ICON::arrangement_to_string(Arrangement a) { return a == Arrangement::ICON_C ? "C" : a == Arrangement::ICON_V ? "V" : a == Arrangement::ICON_E ? "E" : throw SeriousBug("ICON: unsupported arrangement '" + std::to_string(a) + "'", Here()); } void ICON::fill_spec(spec::Custom& custom) const { custom.set("grid", name_); // only T-arrangement is currently catalogued if (auto _uid = uid(); !GridSpecByUID::instance().exists(_uid)) { custom.set("uid", _uid); } } const std::string& ICON::type() const { static const std::string type{"ICON"}; return type; } Grid::uid_type ICON::calculate_uid() const { NOTIMP; } Grid::BoundingBox* ICON::calculate_bbox() const { if (const std::string BOUNDING_BOX = "bounding_box"; catalog().has(BOUNDING_BOX)) { const auto bbox = catalog().get_double_vector(BOUNDING_BOX); ASSERT(bbox.size() == 4); return new BoundingBox{bbox[0], bbox[1], bbox[2], bbox[3]}; } return Unstructured::calculate_bbox(); } Point ICON::first_point() const { ASSERT(!empty()); return PointLonLat{record_.longitudes_.front(), record_.latitudes_.front()}; } Point ICON::last_point() const { ASSERT(!empty()); return PointLonLat{record_.longitudes_.back(), record_.latitudes_.back()}; } static const GridRegisterType GRIDTYPE("ICON"); } // namespace eckit::geo::grid::unstructured eckit-2.0.7/src/eckit/geo/grid/unstructured/UnstructuredGeneric.cc0000664000175000017500000000233615161702250025455 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/unstructured/UnstructuredGeneric.h" #include "eckit/utils/MD5.h" namespace eckit::geo::grid::unstructured { Grid::uid_type UnstructuredGeneric::calculate_uid() const { MD5 hash; hash.add(type()); container()->hash(hash); auto d = hash.digest(); ASSERT(Grid::is_uid(d)); return {d}; } Grid::Spec* UnstructuredGeneric::spec(const std::string& name) { return GridSpecByUID::instance().get(name).spec(); } void UnstructuredGeneric::fill_spec(spec::Custom& custom) const { custom.set("grid", name_); custom.set("uid", uid()); } const std::string& UnstructuredGeneric::type() const { static const std::string type{"unstructured"}; return type; } static const GridRegisterType GRIDTYPE("unstructured"); } // namespace eckit::geo::grid::unstructured eckit-2.0.7/src/eckit/geo/grid/unstructured/FESOM.cc0000664000175000017500000001431715161702250022324 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/unstructured/FESOM.h" #include #include #include #include "eckit/codec/codec.h" #include "eckit/filesystem/PathName.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/geo/cache/Download.h" #include "eckit/geo/cache/Record.h" #include "eckit/geo/container/PointsContainer.h" #include "eckit/geo/util/mutex.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Spec.h" #include "eckit/utils/MD5.h" namespace eckit::geo::util { void hash_vector_double(MD5&, const std::vector&); void hash_vector_size_t(MD5&, const std::vector&); } // namespace eckit::geo::util namespace eckit::geo::grid::unstructured { static util::recursive_mutex MUTEX; class lock_type { util::lock_guard lock_guard_{MUTEX}; }; static const FESOM::FESOMRecord& fesom_record(const spec::Spec& spec) { static cache::Record cache(LibEcKitGeo::cacheDir() + "/grid/fesom"); return cache.get(spec); } FESOM::FESOM(const FESOMRecord& record, const uid_type& uid, const std::string& arrangement, const std::string& name) : Unstructured(new container::PointsLonLatReference{record.longitudes_, record.latitudes_}), name_(name), arrangement_(arrangement_from_string(arrangement)), record_(record) {} FESOM::FESOM(const Spec& spec) : FESOM(fesom_record(spec), spec.get_string("fesom_uid"), spec.get_string("fesom_arrangement"), spec.get_string("name")) { ASSERT(container()); reset_uid(spec.get_string("fesom_uid")); } FESOM::FESOM(uid_type uid) : FESOM(*std::unique_ptr(GridFactory::make_spec(spec::Custom({{"uid", uid}})))) {} FESOM::FESOM(const std::string& name, Arrangement a) : FESOM(*std::unique_ptr( GridFactory::make_spec(spec::Custom({{"grid", name + '_' + arrangement_to_string(a)}})))) {} std::string FESOM::arrangement() const { return arrangement_to_string(arrangement_); } FESOM::FESOMRecord::bytes_t FESOM::FESOMRecord::footprint() const { return sizeof(longitudes_.front()) * longitudes_.size() + sizeof(latitudes_.front()) * latitudes_.size(); } size_t FESOM::FESOMRecord::n() const { return latitudes_.size(); } void FESOM::FESOMRecord::read(const PathName& p) { codec::RecordReader reader(p); uint64_t version = 0; reader.read("version", version).wait(); if (version == 0) { uint64_t n = 0; reader.read("n", n); reader.read("latitude", latitudes_); reader.read("longitude", longitudes_); reader.wait(); ASSERT(n == latitudes_.size()); ASSERT(n == longitudes_.size()); return; } throw SeriousBug("FESOM: unsupported version", Here()); } void FESOM::FESOMRecord::check(const Spec&) const { // FIXME check } Grid::uid_type FESOM::calculate_uid() const { if (arrangement_ == Arrangement::FESOM_N) { MD5 hash; util::hash_vector_double(hash, record_.latitudes_); util::hash_vector_double(hash, record_.longitudes_); auto d = hash.digest(); ASSERT(d.length() == 32); return {d}; } if (arrangement_ == Arrangement::FESOM_C) { // control concurrent reads/writes lock_type lock; static cache::Download download(LibEcKitGeo::cacheDir() + "/grid/fesom"); // bootstrap uid std::unique_ptr spec(GridSpecByUID::instance().get(uid()).spec()); // get coordinates from fesom_arrangement: N auto [lat, lon] = FESOM(spec->get_string("name"), Arrangement::FESOM_N).to_latlons(); // get element indices (0-based) from fesom_arrangement: C (this one) auto url = LibEcKitGeo::url(spec->get_string("url_connectivity")); auto path = download.to_cached_path(url, spec->get_string("name", ""), ".ek"); ASSERT_MSG(path.exists(), "FESOM: file '" + path + "' not found"); codec::RecordReader reader(path); uint64_t version = 0; reader.read("version", version).wait(); ASSERT(version == 0); std::vector elem2d; reader.read("elem2d", elem2d).wait(); MD5 hash; util::hash_vector_double(hash, lat); util::hash_vector_double(hash, lon); util::hash_vector_size_t(hash, elem2d); auto d = hash.digest(); ASSERT(d.length() == 32); return {d}; } NOTIMP; } Grid::Spec* FESOM::spec(const std::string& name) { return GridSpecByUID::instance().get(name).spec(); } Arrangement FESOM::arrangement_from_string(const std::string& str) { return str == "C" ? Arrangement::FESOM_C : str == "N" ? Arrangement::FESOM_N : throw SeriousBug("FESOM: unsupported arrangement '" + str + "'"); } std::string FESOM::arrangement_to_string(Arrangement a) { return a == Arrangement::FESOM_C ? "C" : a == Arrangement::FESOM_N ? "N" : throw SeriousBug("ORCA: unsupported arrangement '" + std::to_string(a) + "'", Here()); } void FESOM::fill_spec(spec::Custom& custom) const { auto grid = name_ + "_" + arrangement_to_string(arrangement_); custom.set("grid", grid); if (auto _uid = uid(); !GridSpecByName::instance().exists(grid) || GridSpecByName::instance().match(grid).spec(grid)->get_string("fesom_uid") != _uid) { custom.set("uid", _uid); } } const std::string& FESOM::type() const { static const std::string type{"FESOM"}; return type; } Point FESOM::first_point() const { ASSERT(!empty()); return PointLonLat{record_.longitudes_.front(), record_.latitudes_.front()}; } Point FESOM::last_point() const { ASSERT(!empty()); return PointLonLat{record_.longitudes_.back(), record_.latitudes_.back()}; } static const GridRegisterType GRIDTYPE("FESOM"); } // namespace eckit::geo::grid::unstructured eckit-2.0.7/src/eckit/geo/grid/unstructured/UnstructuredGeneric.h0000664000175000017500000000223415161702250025314 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/grid/Unstructured.h" namespace eckit::geo::grid::unstructured { class UnstructuredGeneric final : public Unstructured { public: // -- Constructors using Unstructured::Unstructured; // -- Methods const std::string& name() const { return name_; } const std::string& name(const std::string& another) { return name_ = another; } // -- Overridden methods uid_type calculate_uid() const override; const std::string& type() const override; // -- Class methods [[nodiscard]] static Spec* spec(const std::string& name); private: // -- Members std::string name_; // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::grid::unstructured eckit-2.0.7/src/eckit/geo/grid/unstructured/ICON.h0000664000175000017500000000413115161702250022036 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Arrangement.h" #include "eckit/geo/grid/Unstructured.h" namespace eckit { class PathName; } namespace eckit::geo::grid::unstructured { class ICON final : public Unstructured { public: // -- Types struct ICONRecord { explicit ICONRecord() = default; void read(const PathName&); void check(const Spec&) const; using bytes_t = decltype(sizeof(int)); bytes_t footprint() const; size_t n() const; std::vector longitudes_; std::vector latitudes_; }; // -- Constructors explicit ICON(const Spec&); explicit ICON(uid_type); ICON(const std::string& name, Arrangement); // -- Methods std::string name() const { return name_; } std::string arrangement() const; // -- Overridden methods const std::string& type() const override; uid_type calculate_uid() const override; BoundingBox* calculate_bbox() const override; [[nodiscard]] Point first_point() const override; [[nodiscard]] Point last_point() const override; // -- Class methods [[nodiscard]] static Spec* spec(const std::string& name); [[nodiscard]] static Arrangement arrangement_from_string(const std::string&); [[nodiscard]] static std::string arrangement_to_string(Arrangement); private: // -- Constructors ICON(const ICONRecord&, const uid_type&, const std::string& arrangement, const std::string& name); // -- Members std::string name_; Arrangement arrangement_; const ICONRecord& record_; // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::grid::unstructured eckit-2.0.7/src/eckit/geo/grid/unstructured/FESOM.h0000664000175000017500000000410215161702250022155 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/Arrangement.h" #include "eckit/geo/grid/Unstructured.h" namespace eckit { class PathName; } namespace eckit::geo::grid::unstructured { class FESOM final : public Unstructured { public: // -- Types struct FESOMRecord { explicit FESOMRecord() = default; void read(const PathName&); void check(const Spec&) const; using bytes_t = decltype(sizeof(int)); bytes_t footprint() const; size_t n() const; std::vector longitudes_; std::vector latitudes_; }; // -- Constructors explicit FESOM(const Spec&); explicit FESOM(uid_type); FESOM(const std::string& name, Arrangement); // -- Methods std::string name() const { return name_; } std::string arrangement() const; // -- Overridden methods uid_type calculate_uid() const override; const std::string& type() const override; [[nodiscard]] Point first_point() const override; [[nodiscard]] Point last_point() const override; // -- Class methods [[nodiscard]] static Spec* spec(const std::string& name); [[nodiscard]] static Arrangement arrangement_from_string(const std::string&); [[nodiscard]] static std::string arrangement_to_string(Arrangement); private: // -- Constructors FESOM(const FESOMRecord&, const uid_type&, const std::string& arrangement, const std::string& name); // -- Members std::string name_; Arrangement arrangement_; const FESOMRecord& record_; // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::grid::unstructured eckit-2.0.7/src/eckit/geo/grid/regular/0000775000175000017500000000000015161702250020030 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/grid/regular/RegularLL.h0000664000175000017500000000523115161702250022033 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/grid/Regular.h" #include "eckit/geo/range/Regular.h" namespace eckit::geo::grid::regular { class RegularLL final : public Regular { public: // -- Types struct Increments : public std::array { Increments(value_type dlon, value_type dlat) : array{dlon, dlat} {} using array::array; bool operator==(const Increments&) const; bool operator!=(const Increments& other) const { return !operator==(other); } value_type dlon() const { return operator[](0); } value_type dlat() const { return operator[](1); } static Increments make_from_spec(const Spec&); }; struct Reference { static PointLonLat make_from_spec(const Spec&); }; // -- Constructors explicit RegularLL(const Spec&); explicit RegularLL(const Increments&); explicit RegularLL(const Increments&, BoundingBox, PointLonLat ref = {}); // -- Methods [[nodiscard]] static Spec* spec(const std::string& name); [[nodiscard]] PointLonLat reference() const { return {x_.shift(), y_.shift()}; } double dlon() const { return dx(); } double dlat() const { return dy(); } size_t nlon() const { return nx(); } size_t nlat() const { return ny(); } // -- Overridden methods const std::string& type() const override; [[nodiscard]] Point first_point() const override; [[nodiscard]] Point last_point() const override; [[nodiscard]] std::pair, std::vector> to_latlons() const override; [[nodiscard]] Grid* make_grid_cropped(const Area&) const override; [[nodiscard]] BoundingBox* calculate_bbox() const override; double dx() const override { return x_.increment(); } double dy() const override { return y_.increment(); } size_t nx() const override { return x_.size(); } size_t ny() const override { return y_.size(); } const Range& x() const override { return x_; } const Range& y() const override { return y_; } private: // -- Overridden methods void fill_spec(spec::Custom&) const override; // -- Members const range::RegularLongitude x_; const range::RegularLatitude y_; // -- Friends friend class geo::iterator::Regular; }; } // namespace eckit::geo::grid::regular eckit-2.0.7/src/eckit/geo/grid/regular/RegularLL.cc0000664000175000017500000001261715161702250022177 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/regular/RegularLL.h" #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/iterator/Regular.h" #include "eckit/geo/order/Scan.h" #include "eckit/geo/range/Regular.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" #include "eckit/types/Fraction.h" #include "eckit/utils/Translator.h" namespace eckit::geo::grid::regular { #define POSITIVE_REAL "[+]?([0-9]+([.][0-9]*)?|[.][0-9]+)(e[-+][0-9]+)?" static const std::string REGULAR_LL_PATTERN("(" POSITIVE_REAL ")/(" POSITIVE_REAL ")"); #undef POSITIVE_REAL bool RegularLL::Increments::operator==(const Increments& other) const { return types::is_approximately_equal(dlon(), other.dlon()) && types::is_approximately_equal(dlat(), other.dlat()); } RegularLL::Increments RegularLL::Increments::make_from_spec(const Spec& spec) { std::vector grid(2); if (spec.get("dlon", grid[0]) && spec.get("dlat", grid[1])) { return {grid[0], grid[1]}; } if (spec.get("grid", grid) && grid.size() == 2) { return {grid[0], grid[1]}; } throw exception::SpecError("'grid' = ['dlon', 'dlat'] expected", Here()); } PointLonLat RegularLL::Reference::make_from_spec(const Spec& spec) { if (!spec.has("reference_lonlat") || !spec.has("reference_lon") || !spec.has("reference_lat")) { if (spec.has("area") || spec.has("south") || spec.has("west")) { // Default reference point to the south-west corner of the grid if not explicitly provided BoundingBox bbox{spec}; return {bbox.south(), bbox.west()}; } } return PointLonLat::make_from_spec(spec, "reference", {}); } RegularLL::RegularLL(const Spec& spec) : RegularLL(Increments::make_from_spec(spec), BoundingBox{spec}, Reference::make_from_spec(spec)) {} RegularLL::RegularLL(const Increments& inc) : RegularLL(inc, BoundingBox{}, {0, 0}) {} RegularLL::RegularLL(const Increments& inc, BoundingBox bbox, PointLonLat ref) : x_{inc.dlon(), bbox.west(), bbox.east(), ref.lon()}, y_{-inc.dlat(), bbox.north(), bbox.south(), ref.lat()} { ASSERT(!empty()); order(std::string{x().increment() < 0 ? "i-" : "i+"} + // std::string{y().increment() < 0 ? "j-" : "j+"}); // TODO reference to modify bounding box } Grid::Spec* RegularLL::spec(const std::string& name) { std::smatch match; std::regex_match(name, match, std::regex(REGULAR_LL_PATTERN)); ASSERT(match.size() == 9); auto d = Translator{}; return new spec::Custom({{"type", "regular_ll"}, {"grid", std::vector{d(match[1]), d(match[5])}}}); } void RegularLL::fill_spec(spec::Custom& custom) const { custom.set("grid", std::vector{std::abs(dlon()), std::abs(dlat())}); if (auto o = order(); o != order::Scan::order_default()) { custom.set("order", o); } if (const auto first = first_point(); !points_equal(first, PointLonLat{x_.a(), y_.a()})) { const auto ref{reference()}; custom.set("reference", std::vector{ref.lon(), ref.lat()}); } if (auto bbox = boundingBox(); bbox != BoundingBox::bounding_box_default()) { auto [n, w, s, e] = bbox.deconstruct(); custom.set("area", std::vector{n, w, s, e}); } } const std::string& RegularLL::type() const { static const std::string type{"regular-ll"}; return type; } Point RegularLL::first_point() const { ASSERT(!empty()); return PointLonLat{x().values().front(), y().values().front()}; // First longitude and first latitude } Point RegularLL::last_point() const { ASSERT(!empty()); return PointLonLat{x().values().back(), y().values().back()}; } std::pair, std::vector> RegularLL::to_latlons() const { const auto N = size(); std::pair, std::vector> latlon; auto& lat = latlon.first; auto& lon = latlon.second; lat.reserve(N); lon.reserve(N); for (auto point : *this) { const auto& p = std::get(point); lat.emplace_back(p.lat()); lon.emplace_back(p.lon()); } return latlon; } Grid* RegularLL::make_grid_cropped(const Area& crop) const { if (auto cropped(boundingBox()); crop.intersects(cropped)) { return new RegularLL({dlon(), dlat()}, cropped); } throw UserError("RegularLL: cannot crop grid (empty intersection)", Here()); } Grid::BoundingBox* RegularLL::calculate_bbox() const { auto n = y_.includesNorthPole() ? PointLonLat::RIGHT_ANGLE : y_.max(); auto s = y_.includesSouthPole() ? -PointLonLat::RIGHT_ANGLE : y_.min(); auto w = x_.a(); auto e = x_.periodic() ? w + PointLonLat::FULL_ANGLE : x_.b(); return new BoundingBox{n, w, s, e}; } static const auto GRIDNAME = GridRegisterName(REGULAR_LL_PATTERN); static const GridRegisterType GRIDTYPE1("regular_ll"); static const GridRegisterType GRIDTYPE2("rotated_ll"); } // namespace eckit::geo::grid::regular eckit-2.0.7/src/eckit/geo/grid/regular/RegularGaussian.h0000664000175000017500000000343515161702250023302 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/grid/Regular.h" #include "eckit/geo/range/GaussianLatitude.h" #include "eckit/geo/range/Regular.h" namespace eckit::geo::grid::regular { class RegularGaussian final : public Regular { public: // -- Constructors explicit RegularGaussian(const Spec&); explicit RegularGaussian(size_t N, BoundingBox = {}); // -- Methods size_t N() const { return N_; } // -- Overridden methods const std::string& type() const override; [[nodiscard]] Point first_point() const override; [[nodiscard]] Point last_point() const override; [[nodiscard]] BoundingBox* calculate_bbox() const override; [[nodiscard]] Grid* make_grid_cropped(const Area&) const override; double dx() const override { return x_.increment(); } double dy() const override { return y_.increment(); } size_t nx() const override { return x_.size(); } size_t ny() const override { return y_.size(); } const Range& x() const override { return x_; } const Range& y() const override { return y_; } // -- Class members [[nodiscard]] static Spec* spec(const std::string& name); private: // -- Members const size_t N_; const range::RegularLongitude x_; const range::GaussianLatitude y_; // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::grid::regular eckit-2.0.7/src/eckit/geo/grid/regular/RegularXY.h0000664000175000017500000000400015161702250022055 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/area/BoundingBoxXY.h" #include "eckit/geo/grid/Regular.h" #include "eckit/geo/range/Regular.h" namespace eckit::geo::grid::regular { class RegularXY : public Regular { public: // -- Types struct Increments : public std::array { Increments(value_type dx, value_type dy) : array{dx, dy} {} using array::array; bool operator==(const Increments&) const; bool operator!=(const Increments& other) const { return !operator==(other); } const value_type& dx = array::operator[](0); const value_type& dy = array::operator[](1); static Increments make_from_spec(const Spec&); }; using BoundingBoxXY = area::BoundingBoxXY; // -- Constructors explicit RegularXY(const Spec&); explicit RegularXY(const Increments&, BoundingBoxXY); // -- Overridden methods const std::string& type() const override; [[nodiscard]] Point first_point() const override; [[nodiscard]] Point last_point() const override; double dx() const override { return x_.increment(); } double dy() const override { return y_.increment(); } size_t nx() const override { return x_.size(); } size_t ny() const override { return y_.size(); } const Range& x() const override { return x_; } const Range& y() const override { return y_; } private: // -- Overriden methods void fill_spec(spec::Custom&) const override; // -- Members const range::RegularXY x_; const range::RegularXY y_; // -- Friends friend class geo::iterator::Regular; }; } // namespace eckit::geo::grid::regular eckit-2.0.7/src/eckit/geo/grid/regular/RegularXY.cc0000664000175000017500000000637715161702250022236 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/regular/RegularXY.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/Projection.h" #include "eckit/geo/Shape.h" #include "eckit/geo/range/Regular.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::grid::regular { namespace { Range* make_x_range(const spec::Spec& spec) { auto inc = RegularXY::Increments::make_from_spec(spec); auto first = PointLonLat::make_from_spec(spec, "first"); Shape shape(spec); auto a = std::get(std::unique_ptr(ProjectionFactory::build(spec))->inv(first)).X(); auto b = a + (shape.nx > 1 ? inc.dx * static_cast(shape.nx - 1) : 0); return new range::RegularXY(inc.dx, a, b, a); } Range* make_y_range(const spec::Spec& spec) { auto inc = RegularXY::Increments::make_from_spec(spec); auto first = PointLonLat::make_from_spec(spec, "first"); Shape shape(spec); auto a = std::get(std::unique_ptr(ProjectionFactory::build(spec))->inv(first)).Y(); auto b = a + (shape.ny > 1 ? inc.dy * static_cast(shape.ny - 1) : 0); return new range::RegularXY(inc.dy, a, b, a); } } // namespace bool RegularXY::Increments::operator==(const Increments& other) const { return types::is_approximately_equal(dx, other.dx) && types::is_approximately_equal(dy, other.dy); } RegularXY::Increments RegularXY::Increments::make_from_spec(const Spec& spec) { std::vector grid(2); if (spec.get("dx", grid[0]) && spec.get("dy", grid[1])) { return {grid[0], grid[1]}; } if (spec.get("grid", grid) && grid.size() == 2) { return {grid[0], -grid[1]}; } throw exception::SpecError("'grid' = ['dx', 'dy'] expected", Here()); } RegularXY::RegularXY(const Spec& spec) : RegularXY(Increments::make_from_spec(spec), BoundingBoxXY(spec)) {} RegularXY::RegularXY(const Increments& inc, BoundingBoxXY bbox) : x_(inc.dx, bbox.min_x, bbox.max_x), y_(inc.dy, bbox.min_y, bbox.max_y) { ASSERT(!empty()); } const std::string& RegularXY::type() const { NOTIMP; } Point RegularXY::first_point() const { ASSERT(!empty()); return PointXY{x().values().front(), y().values().front()}; } Point RegularXY::last_point() const { ASSERT(!empty()); return PointXY{x().values().back(), y().values().back()}; } void RegularXY::fill_spec(spec::Custom& custom) const { Regular::fill_spec(custom); custom.set("grid", std::vector{dx(), dy()}); custom.set("shape", std::vector{static_cast(nx()), static_cast(ny())}); // custom.set("first_lonlat", std::vector{first_lonlat.lon, first_lonlat.lat}); } static const GridRegisterType GRID1("lambert"); static const GridRegisterType GRID2("lambert_lam"); } // namespace eckit::geo::grid::regular eckit-2.0.7/src/eckit/geo/grid/regular/RegularGaussian.cc0000664000175000017500000000611715161702250023440 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/regular/RegularGaussian.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/PointLonLat.h" #include "eckit/geo/range/GaussianLatitude.h" #include "eckit/geo/range/Regular.h" #include "eckit/spec/Custom.h" #include "eckit/utils/Translator.h" namespace eckit::geo::grid::regular { namespace { size_t check_N(size_t N) { if (N == 0) { throw exception::GridError("RegularGaussian: Gaussian N cannot be zero", Here()); } return N; } } // namespace RegularGaussian::RegularGaussian(const Spec& spec) : RegularGaussian(spec.get_unsigned("N"), BoundingBox(spec)) {} RegularGaussian::RegularGaussian(size_t N, BoundingBox bbox) : N_(check_N(N)), x_(*range::RegularLongitude(360. / static_cast(4 * N), 0., 360.) .make_cropped_range(bbox.west(), bbox.east())), y_(*range::GaussianLatitude(N, false).make_cropped_range(bbox.north(), bbox.south())) { ASSERT(!empty()); } Grid::Spec* RegularGaussian::spec(const std::string& name) { ASSERT(name.size() > 1 && (name[0] == 'f' || name[0] == 'F')); auto N = Translator{}(name.substr(1)); return new spec::Custom({{"type", "regular_gg"}, {"N", N}}); } void RegularGaussian::fill_spec(spec::Custom& custom) const { Regular::fill_spec(custom); custom.set("grid", "F" + std::to_string(N_)); } const std::string& RegularGaussian::type() const { static const std::string type{"regular-gg"}; return type; } Point RegularGaussian::first_point() const { ASSERT(!empty()); return PointLonLat{x().values().front(), y().values().front()}; } Point RegularGaussian::last_point() const { ASSERT(!empty()); return PointLonLat{x().values().back(), y().values().back()}; } Grid::BoundingBox* RegularGaussian::calculate_bbox() const { return new BoundingBox{y_.includesNorthPole() ? PointLonLat::RIGHT_ANGLE : y_.max(), // x_.min(), // y_.includesSouthPole() ? -PointLonLat::RIGHT_ANGLE : y_.min(), // x_.periodic() ? x_.min() + PointLonLat::FULL_ANGLE : x_.max()}; } Grid* RegularGaussian::make_grid_cropped(const Area& crop) const { if (auto cropped(boundingBox()); crop.intersects(cropped)) { return new RegularGaussian(N_, cropped); } throw UserError("RegularGaussian: cannot crop grid (empty intersection)", Here()); } static const auto GRIDNAME = GridRegisterName("f[1-9][0-9]*"); static const GridRegisterType GRIDTYPE1("regular_gg"); static const GridRegisterType GRIDTYPE2("rotated_gg"); } // namespace eckit::geo::grid::regular eckit-2.0.7/src/eckit/geo/grid/SphericalHarmonics.cc0000664000175000017500000000501715161702250022457 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/SphericalHarmonics.h" #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/area/None.h" #include "eckit/spec/Custom.h" #include "eckit/utils/MD5.h" #include "eckit/utils/Translator.h" namespace eckit::geo::grid { static const std::string NOT_SUPPORTED{"SphericalHarmonics does not support gridded functionality"}; static const std::string PATTERN{"[tT]([1-9][0-9]*)"}; static const GridRegisterType GRIDTYPE("sh"); static const auto GRIDNAME = GridRegisterName(PATTERN); SphericalHarmonics::SphericalHarmonics(const Spec& spec) : truncation_(spec.get_unsigned("truncation")) {} SphericalHarmonics::SphericalHarmonics(size_t T) : truncation_(T) {} Grid::Spec* SphericalHarmonics::spec(const std::string& name) { std::smatch match; std::regex_match(name, match, std::regex(PATTERN)); ASSERT(match.size() == 2); auto u = Translator{}; return new spec::Custom({{"type", "sh"}, {"truncation", u(match[1])}}); } Grid::iterator SphericalHarmonics::cbegin() const { throw exception::GridError(NOT_SUPPORTED, Here()); } Grid::iterator SphericalHarmonics::cend() const { throw exception::GridError(NOT_SUPPORTED, Here()); } const std::string& SphericalHarmonics::type() const { static const std::string TYPE{"sh"}; return TYPE; } std::vector SphericalHarmonics::shape() const { return {size()}; } bool SphericalHarmonics::empty() const { return size() == 0; } const Area& SphericalHarmonics::area() const { static const geo::area::None NONE; return NONE; } [[nodiscard]] Grid::uid_type SphericalHarmonics::calculate_uid() const { return (MD5{} << type() << truncation()).digest(); } [[nodiscard]] Grid::BoundingBox* SphericalHarmonics::calculate_bbox() const { return new BoundingBox; } void SphericalHarmonics::fill_spec(spec::Custom& custom) const { custom.set("grid", "T" + std::to_string(truncation_)); } size_t SphericalHarmonics::number_of_complex_coefficients(size_t truncation) { return (truncation + 1) * (truncation + 2) / 2; } } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/grid/SphericalHarmonics.h0000664000175000017500000000323115161702250022315 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Grid.h" namespace eckit::geo::grid { class SphericalHarmonics final : public Grid { public: // -- Constructors explicit SphericalHarmonics(const Spec&); explicit SphericalHarmonics(size_t T); // -- Methods [[nodiscard]] static Spec* spec(const std::string& name); size_t truncation() const { return truncation_; } // -- Overridden methods iterator cbegin() const override; iterator cend() const override; const std::string& type() const override; std::vector shape() const override; bool empty() const override; size_t size() const override { return number_of_complex_coefficients(truncation_); } [[nodiscard]] uid_type calculate_uid() const override; const Area& area() const override; bool includesNorthPole() const override { return true; } bool includesSouthPole() const override { return true; } bool isPeriodicWestEast() const override { return true; } [[nodiscard]] BoundingBox* calculate_bbox() const override; void fill_spec(spec::Custom&) const override; // -- Class methods static size_t number_of_complex_coefficients(size_t truncation); private: // -- Members size_t truncation_; }; } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/grid/ORCA.h0000664000175000017500000000657415161702250017300 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include "eckit/geo/Arrangement.h" #include "eckit/geo/Grid.h" #include "eckit/geo/container/PointsContainer.h" namespace eckit { class PathName; } namespace eckit::geo::grid { class ORCA final : public Grid { public: // -- Types struct ORCARecord { explicit ORCARecord() = default; void read(const PathName&); void check(const Spec&) const; size_t write(const PathName&, const std::string& compression = "none"); uid_type calculate_uid(Arrangement) const; using bytes_t = decltype(sizeof(int)); bytes_t footprint() const; size_t ni() const; size_t nj() const; std::array dimensions_ = {-1, -1}; std::array halo_ = {-1, -1, -1, -1}; std::array pivot_ = {-1, -1}; std::vector longitudes_; std::vector latitudes_; std::vector flags_; }; // -- Constructors explicit ORCA(const Spec&); explicit ORCA(uid_type); ORCA(const std::string& name, Arrangement); // -- Methods size_t nx() const { return record_.nj(); } size_t ny() const { return record_.ni(); } std::string name() const { return name_; } std::string arrangement() const; std::shared_ptr container() const { return container_; } // -- Overridden methods iterator cbegin() const override; iterator cend() const override; uid_type calculate_uid() const override; const std::string& type() const override; std::vector shape() const override { return {record_.ni(), record_.nj()}; } size_t size() const override { return record_.ni() * record_.nj(); }; BoundingBox* calculate_bbox() const override; bool includesNorthPole() const override { return true; } bool includesSouthPole() const override { return true; } // FIXME: not sure this is semanticaly correct bool isPeriodicWestEast() const override { return true; } [[nodiscard]] Point first_point() const override; [[nodiscard]] Point last_point() const override; [[nodiscard]] std::vector to_points() const override; [[nodiscard]] std::pair, std::vector> to_latlons() const override; const order_type& order() const override; renumber_type reorder(const order_type& to) const override; // -- Class methods [[nodiscard]] static Spec* spec_from_uid(const std::string& name); [[nodiscard]] static Arrangement arrangement_from_string(const std::string&); [[nodiscard]] static std::string arrangement_to_string(Arrangement); private: // -- Members std::string name_; Arrangement arrangement_; const ORCARecord& record_; std::shared_ptr container_; // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/grid/reduced/0000775000175000017500000000000015161702250020002 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/grid/reduced/HEALPix.cc0000664000175000017500000001243315161702250021506 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/reduced/HEALPix.h" #include #include #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/iterator/Unstructured.h" #include "eckit/geo/util.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" #include "eckit/utils/SafeCasts.h" namespace eckit::geo::grid::reduced { static const std::string HEALPIX_PATTERN = "h([rn][1-9][0-9]*|[1-9][0-9]*(|r|_ring|n|_nested))"; HEALPix::HEALPix(const Spec& spec) : HEALPix(into_unsigned(spec.get_long("Nside")), spec.get_string("order", order::HEALPix::order_default())) { // NOTE: this is format-specific and should be moved from here if (double west = 45.; spec.get("west", west) && !types::is_approximately_equal(west, 45., PointLonLat::EPS)) { throw exception::GridError("HEALPix: west offset should be 45 degrees", Here()); } } HEALPix::HEALPix(size_t Nside, order_type order) : Nside_(Nside), healpix_(order) { boundingBox(new BoundingBox{}); if (Nside_ == 0) { throw exception::GridError("HEALPix: Nside must be greater than zero", Here()); } } Grid::iterator HEALPix::cbegin() const { std::ignore = to_points(); ASSERT(points_); return iterator{new geo::iterator::Unstructured(*this, 0, points_)}; } Grid::iterator HEALPix::cend() const { return iterator{new geo::iterator::Unstructured(*this)}; } size_t HEALPix::nx(size_t j) const { ASSERT(j < ny()); return j < Nside_ ? 4 * (j + 1) : j < 3 * Nside_ ? 4 * Nside_ : nx(ny() - 1 - j); } size_t HEALPix::ny() const { return 4 * Nside_ - 1; } Grid::Spec* HEALPix::spec(const std::string& name) { static const std::regex rex("[1-9][0-9]*"); std::smatch match; ASSERT(std::regex_search(name, match, rex)); auto Nside = std::stoul(match.str()); auto nested = (name.find("n") != std::string::npos || name.find("N") != std::string::npos) && (name.find("r") == std::string::npos && name.find("R") == std::string::npos); return new spec::Custom{ {"type", "HEALPix"}, {"Nside", Nside}, {"order", nested ? order::HEALPix::NESTED : order::HEALPix::RING}}; } size_t HEALPix::size_from_nside(size_t Nside) { return 12 * Nside * Nside; } size_t HEALPix::nside_from_size(size_t size) { if (auto Nside = static_cast(std::sqrt(size / 12)); size == size_from_nside(Nside)) { return Nside; } throw exception::GridError("HEALPix: invalid size: " + std::to_string(size), Here()); } size_t HEALPix::size() const { return size_from_nside(Nside_); } std::vector HEALPix::to_points() const { if (!points_) { // reorder to this grid's order const auto ren = order::HEALPix{}.reorder(order(), Nside()); std::vector points(size()); const auto& lats = latitudes(); for (size_t j = 0, k = 0; j < lats.size(); ++j) { for (const auto lon : longitudes(j)) { points[ren.at(k++)] = PointLonLat{lon, lats[j]}; } } points_ = std::make_shared(std::move(points)); } ASSERT(points_); return points_->to_points(); } const std::vector& HEALPix::latitudes() const { const auto Nj = ny(); if (latitudes_.empty()) { latitudes_.resize(Nj); auto i = latitudes_.begin(); auto j = latitudes_.rbegin(); for (size_t ring = 1; ring < 2 * Nside_; ++ring, ++i, ++j) { const auto f = ring < Nside_ ? 1. - static_cast(ring * ring) / (3 * static_cast(Nside_ * Nside_)) : 4. / 3. - 2 * static_cast(ring) / (3 * static_cast(Nside_)); *i = 90. - util::RADIAN_TO_DEGREE * std::acos(f); *j = -*i; } *i = 0.; } ASSERT(latitudes_.size() == Nj); return latitudes_; } std::vector HEALPix::longitudes(size_t j) const { const auto Ni = nx(j); const auto step = 360. / static_cast(Ni); const auto start = j < Nside_ || 3 * Nside_ - 1 < j || static_cast((j + Nside_) % 2) ? step / 2. : 0.; std::vector lons(Ni); std::generate_n(lons.begin(), Ni, [start, step, n = 0ULL]() mutable { return start + static_cast(n++) * step; }); return lons; } void HEALPix::fill_spec(spec::Custom& custom) const { custom.set("grid", "H" + std::to_string(Nside_)); if (const auto o = order(); o != order::HEALPix::order_default()) { custom.set("order", o); } } const std::string& HEALPix::type() const { static const std::string type{"healpix"}; return type; } static const GridRegisterType GRIDTYPE1("HEALPix"); static const GridRegisterType GRIDTYPE2("healpix"); static const auto GRIDNAME = GridRegisterName(HEALPIX_PATTERN); } // namespace eckit::geo::grid::reduced eckit-2.0.7/src/eckit/geo/grid/reduced/ReducedGaussian.h0000664000175000017500000000414215161702250023222 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/geo/Range.h" #include "eckit/geo/grid/Reduced.h" #include "eckit/geo/order/Scan.h" #include "eckit/geo/util.h" namespace eckit::geo::grid::reduced { class ReducedGaussian : public Reduced { public: // -- Constructors explicit ReducedGaussian(const Spec&); explicit ReducedGaussian(const pl_type&, const BoundingBox& = BoundingBox::bounding_box_default()); explicit ReducedGaussian(size_t N, const BoundingBox& = BoundingBox::bounding_box_default()); explicit ReducedGaussian(size_t N, const pl_type&, const BoundingBox& = BoundingBox::bounding_box_default()); // -- Methods size_t N() const { return N_; } const pl_type& pl() const { return pl_; } // -- Overridden methods iterator cbegin() const override; iterator cend() const override; size_t size() const override; size_t nx(size_t j) const override; size_t ny() const override; const order_type& order() const override { return scan_.order(); } renumber_type reorder(const order_type& to) const override { return scan_.reorder(to, pl_); } const std::vector& latitudes() const override; std::vector longitudes(size_t j) const override; private: // -- Members const size_t N_; const pl_type pl_; size_t j_; size_t Nj_; mutable std::vector> longitude_; std::unique_ptr latitude_; order::Scan scan_; // -- Overridden methods void fill_spec(spec::Custom&) const override; const std::string& type() const override; [[nodiscard]] Grid* make_grid_cropped(const Area&) const override; }; } // namespace eckit::geo::grid::reduced eckit-2.0.7/src/eckit/geo/grid/reduced/ReducedGaussian.cc0000664000175000017500000001233115161702250023357 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/reduced/ReducedGaussian.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/iterator/Reduced.h" #include "eckit/geo/range/GaussianLatitude.h" #include "eckit/geo/range/Regular.h" #include "eckit/spec/Custom.h" #include "eckit/utils/Translator.h" namespace eckit::geo::grid::reduced { namespace { size_t check_N(size_t N) { if (N == 0) { throw exception::GridError("ReducedGaussian: Gaussian N cannot be zero", Here()); } return N; } Range* make_x_range(size_t Ni, const area::BoundingBox& bbox) { if (Ni == 0) { throw exception::GridError("ReducedGaussian: zero points in x range (along parallel)", Here()); } range::RegularLongitude global(360. / static_cast(Ni), 0., 360.); return global.make_cropped_range(bbox.west(), bbox.east()); } Range* make_y_range(size_t N, const area::BoundingBox& bbox) { if (N == 0) { throw exception::GridError("ReducedGaussian: zero points in y range (along meridian)", Here()); } range::GaussianLatitude global(N, false); return global.make_cropped_range(bbox.north(), bbox.south()); } } // namespace ReducedGaussian::ReducedGaussian(const Spec& spec) : ReducedGaussian(spec.get_long_vector("pl"), BoundingBox{spec}) {} ReducedGaussian::ReducedGaussian(const pl_type& pl, const BoundingBox& bbox) : ReducedGaussian(pl.size() / 2, pl, bbox) {} ReducedGaussian::ReducedGaussian(size_t N, const pl_type& pl, const BoundingBox& bbox) : N_(check_N(N)), pl_(pl), j_(0), Nj_(pl.size()), longitude_(Nj_) { ASSERT(N_ * 2 == pl_.size()); ASSERT(0 < N_ && Nj_ <= 2 * N_); boundingBox(new BoundingBox{bbox}); latitude_.reset(make_y_range(N, bbox)); ASSERT(latitude_); } ReducedGaussian::ReducedGaussian(size_t N, const BoundingBox& bbox) : ReducedGaussian(N, util::reduced_octahedral_pl(N), bbox) {} Grid::iterator ReducedGaussian::cbegin() const { return iterator{new geo::iterator::Reduced(*this, 0)}; } Grid::iterator ReducedGaussian::cend() const { return iterator{new geo::iterator::Reduced(*this, size())}; } size_t ReducedGaussian::size() const { return nxacc().back(); } size_t ReducedGaussian::nx(size_t j) const { if (!longitude_.at(j_ + j)) { const auto& bbox = boundingBox(); auto Ni = pl_.at(j_ + j); ASSERT(Ni >= 0); longitude_[j].reset(make_x_range(static_cast(Ni), bbox)); ASSERT(longitude_[j]); } return longitude_[j]->size(); } size_t ReducedGaussian::ny() const { return latitude_->size(); } const std::vector& ReducedGaussian::latitudes() const { return latitude_->values(); } std::vector ReducedGaussian::longitudes(size_t j) const { if (nx(j) > 0) { ASSERT(longitude_[j]); return longitude_[j]->values(); } return {}; } void ReducedGaussian::fill_spec(spec::Custom& custom) const { Reduced::fill_spec(custom); if (pl_ == util::reduced_octahedral_pl(N_)) { custom.set("grid", "O" + std::to_string(N_)); } else { custom.set("grid", "N" + std::to_string(N_)); if (!util::reduced_classical_pl_known(N_)) { custom.set("pl", pl_); } } if (order() != order::Scan::order_default()) { custom.set("order", order()); } } const std::string& ReducedGaussian::type() const { static const std::string type{"reduced-gg"}; return type; } Grid* ReducedGaussian::make_grid_cropped(const Area& crop) const { if (auto cropped(boundingBox()); crop.intersects(cropped)) { return new ReducedGaussian(N_, pl_, BoundingBox{cropped.north(), cropped.west(), cropped.south(), cropped.east()}); } throw UserError("ReducedGaussian: cannot crop grid (empty intersection)", Here()); } struct ReducedGaussianClassical { static Grid::Spec* spec(const std::string& name) { ASSERT(name.size() > 1 && (name[0] == 'n' || name[0] == 'N')); auto N = Translator{}(name.substr(1)); return new spec::Custom({{"type", "reduced_gg"}, {"N", N}, {"pl", util::reduced_classical_pl(N)}}); } }; struct ReducedGaussianOctahedral { static Grid::Spec* spec(const std::string& name) { ASSERT(name.size() > 1 && (name[0] == 'o' || name[0] == 'O')); auto N = Translator{}(name.substr(1)); return new spec::Custom({{"type", "reduced_gg"}, {"N", N}, {"pl", util::reduced_octahedral_pl(N)}}); } }; static const auto GRIDNAME1 = GridRegisterName("n[1-9][0-9]*"); static const auto GRIDNAME2 = GridRegisterName("o[1-9][0-9]*"); static const GridRegisterType GRIDTYPE1("reduced_gg"); static const GridRegisterType GRIDTYPE2("reduced_rotated_gg"); } // namespace eckit::geo::grid::reduced eckit-2.0.7/src/eckit/geo/grid/reduced/HEALPix.h0000664000175000017500000000441215161702250021346 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/container/PointsContainer.h" #include "eckit/geo/grid/Reduced.h" #include "eckit/geo/order/HEALPix.h" namespace eckit::geo::grid::reduced { class HEALPix final : public Reduced { public: // -- Constructors explicit HEALPix(const Spec&); explicit HEALPix(size_t Nside, order_type = order::HEALPix::order_default()); // -- Methods size_t Nside() const { return Nside_; } // -- Overridden methods iterator cbegin() const override; iterator cend() const override; size_t size() const override; size_t nx(size_t j) const override; size_t ny() const override; [[nodiscard]] std::vector to_points() const override; const order_type& order() const override { return healpix_.order(); } renumber_type reorder(const order_type& to) const override { return healpix_.reorder(to, Nside_); } [[nodiscard]] Grid* make_grid_reordered(const order_type& order) const override { return new HEALPix(Nside_, order); } const std::vector& latitudes() const override; std::vector longitudes(size_t j) const override; // -- Class members [[nodiscard]] static Spec* spec(const std::string& name); // -- Class methods static size_t size_from_nside(size_t); static size_t nside_from_size(size_t); private: // -- Members const size_t Nside_; order::HEALPix healpix_; mutable std::shared_ptr points_; mutable std::vector latitudes_; // -- Overridden methods bool includesNorthPole() const override { return true; } bool includesSouthPole() const override { return true; } bool isPeriodicWestEast() const override { return true; } void fill_spec(spec::Custom&) const override; const std::string& type() const override; }; } // namespace eckit::geo::grid::reduced eckit-2.0.7/src/eckit/geo/grid/reduced/ReducedLonLat.cc0000664000175000017500000000735615161702250023011 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/reduced/ReducedLonLat.h" #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/iterator/Reduced.h" #include "eckit/geo/range/Regular.h" #include "eckit/spec/Custom.h" #include "eckit/utils/SafeCasts.h" namespace eckit::geo::grid::reduced { namespace { range::RegularLongitude* make_lon_range(size_t Ni, const area::BoundingBox& bbox) { const auto west = bbox.west(); const auto east = bbox.periodic() ? west + PointLonLat::FULL_ANGLE : bbox.east(); if (Ni == 0) { return range::RegularLongitude::make_empty_range(west, east); } ASSERT(2 <= Ni); ASSERT(west < east); return new range::RegularLongitude((east - west) / static_cast(bbox.periodic() ? Ni : Ni - 1), west, east, west); } range::RegularLatitude* make_lat_range(size_t Nj, const area::BoundingBox& bbox) { const auto north = bbox.north(); const auto south = bbox.south(); ASSERT(2 <= Nj); ASSERT(south < north); return new range::RegularLatitude((south - north) / static_cast(Nj - 1), north, south, south); } } // namespace ReducedLonLat::ReducedLonLat(const Spec& spec) : ReducedLonLat(spec.get_long_vector("pl"), BoundingBox{spec}) {} ReducedLonLat::ReducedLonLat(const pl_type& pl, const BoundingBox& bbox) : pl_(pl), longitude_(pl.size()) { if (pl_.empty()) { throw exception::GridError("ReducedLonLat: 'pl' must not be empty", Here()); } auto pl_periodic = [&]() { auto max_pl = *std::max_element(pl.begin(), pl.end()); ASSERT(max_pl >= 2); auto inc = PointLonLat::FULL_ANGLE / static_cast(max_pl - 1); return range::RegularLongitude(inc, bbox.west(), bbox.east(), bbox.west()).periodic(); }; boundingBox( new BoundingBox{bbox.north(), bbox.west(), bbox.south(), bbox.periodic() || pl_periodic() ? bbox.west() + PointLonLat::FULL_ANGLE : bbox.east()}); latitude_.reset(make_lat_range(pl.size(), bbox)); ASSERT(latitude_); } Grid::iterator ReducedLonLat::cbegin() const { return iterator{new geo::iterator::Reduced(*this, 0)}; } Grid::iterator ReducedLonLat::cend() const { return iterator{new geo::iterator::Reduced(*this, size())}; } size_t ReducedLonLat::size() const { return nxacc().back(); } size_t ReducedLonLat::nx(size_t j) const { if (!longitude_.at(j)) { longitude_[j].reset(make_lon_range(into_unsigned(pl_.at(j)), boundingBox())); ASSERT(longitude_[j]); } return longitude_[j]->size(); } size_t ReducedLonLat::ny() const { return latitude_->size(); } const std::vector& ReducedLonLat::latitudes() const { return latitude_->values(); } std::vector ReducedLonLat::longitudes(size_t j) const { if (nx(j) > 0) { ASSERT(longitude_[j]); return longitude_[j]->values(); } return {}; } void ReducedLonLat::fill_spec(spec::Custom& custom) const { Reduced::fill_spec(custom); custom.set("pl", pl_); if (order() != order::Scan::order_default()) { custom.set("order", order()); } } const std::string& ReducedLonLat::type() const { static const std::string type{"reduced-ll"}; return type; } static const GridRegisterType GRIDTYPE("reduced_ll"); } // namespace eckit::geo::grid::reduced eckit-2.0.7/src/eckit/geo/grid/reduced/ReducedLonLat.h0000664000175000017500000000340115161702250022636 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/grid/Reduced.h" #include "eckit/geo/order/Scan.h" #include "eckit/geo/range/Regular.h" #include "eckit/geo/util.h" namespace eckit::geo::grid::reduced { class ReducedLonLat : public Reduced { public: // -- Constructors explicit ReducedLonLat(const Spec&); explicit ReducedLonLat(const pl_type&, const BoundingBox& = BoundingBox::bounding_box_default()); // -- Methods const pl_type& pl() const { return pl_; } // -- Overridden methods iterator cbegin() const override; iterator cend() const override; size_t size() const override; size_t nx(size_t j) const override; size_t ny() const override; const order_type& order() const override { return scan_.order(); } renumber_type reorder(const order_type& to) const override { return scan_.reorder(to, pl_); } const std::vector& latitudes() const override; std::vector longitudes(size_t j) const override; private: // -- Members const pl_type pl_; mutable std::vector> longitude_; std::unique_ptr latitude_; order::Scan scan_; // -- Overridden methods void fill_spec(spec::Custom&) const override; const std::string& type() const override; }; } // namespace eckit::geo::grid::reduced eckit-2.0.7/src/eckit/geo/grid/ORCA.cc0000664000175000017500000001744415161702250017434 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/ORCA.h" #include #include "eckit/codec/codec.h" #include "eckit/filesystem/PathName.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/geo/cache/Record.h" #include "eckit/geo/iterator/Unstructured.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Spec.h" #include "eckit/types/FloatCompare.h" #include "eckit/utils/MD5.h" namespace eckit::geo::util { void hash_vector_double(MD5&, const std::vector&); } namespace eckit::geo::grid { static const ORCA::ORCARecord& orca_record(const spec::Spec& spec) { static cache::Record cache(LibEcKitGeo::cacheDir() + "/grid/orca"); return cache.get(spec); } ORCA::ORCA(const Spec& spec) : name_(spec.get_string("name")), arrangement_(arrangement_from_string(spec.get_string("orca_arrangement"))), record_(orca_record(spec)), container_(new container::PointsLonLatReference{record_.longitudes_, record_.latitudes_}) { ASSERT(container_); if (spec.has("orca_uid")) { reset_uid(spec.get_string("orca_uid")); } } ORCA::ORCA(uid_type uid) : ORCA(*std::unique_ptr(GridFactory::make_spec(spec::Custom({{"uid", uid}})))) {} ORCA::ORCA(const std::string& name, Arrangement a) : ORCA(*std::unique_ptr( GridFactory::make_spec(spec::Custom({{"grid", name + '_' + arrangement_to_string(a)}})))) {} std::string ORCA::arrangement() const { return arrangement_to_string(arrangement_); } Grid::uid_type ORCA::ORCARecord::calculate_uid(Arrangement arrangement) const { MD5 hash; hash.add(arrangement_to_string(arrangement)); util::hash_vector_double(hash, latitudes_); util::hash_vector_double(hash, longitudes_); auto d = hash.digest(); ASSERT(Grid::is_uid(d)); return {d}; } ORCA::ORCARecord::bytes_t ORCA::ORCARecord::footprint() const { return sizeof(dimensions_.front()) * dimensions_.size() + sizeof(halo_.front()) * halo_.size() + sizeof(pivot_.front()) * pivot_.size() + sizeof(longitudes_.front()) * longitudes_.size() + sizeof(latitudes_.front()) * latitudes_.size() + sizeof(flags_.front()) * flags_.size(); } size_t ORCA::ORCARecord::ni() const { ASSERT(0 <= dimensions_[0]); return static_cast(dimensions_[0]); } size_t ORCA::ORCARecord::nj() const { ASSERT(0 <= dimensions_[1]); return static_cast(dimensions_[1]); } void ORCA::ORCARecord::read(const PathName& p) { codec::RecordReader reader(p); int version = -1; reader.read("version", version).wait(); if (version == 0) { reader.read("dimensions", dimensions_); reader.read("pivot", pivot_); // different order from writer reader.read("halo", halo_); //... reader.read("longitude", longitudes_); reader.read("latitude", latitudes_); reader.read("flags", flags_); reader.wait(); return; } throw SeriousBug("ORCA::read: unsupported version", Here()); } void ORCA::ORCARecord::check(const Spec& spec) const { if (spec.get_bool("orca_uid_check", false)) { auto uid = spec.get_string("orca_uid"); ASSERT(Grid::is_uid(uid)); ASSERT(uid == calculate_uid(arrangement_from_string(spec.get_string("orca_arrangement")))); } if (std::vector d; spec.get("dimensions", d)) { ASSERT(d.size() == 2); ASSERT(d[0] == dimensions_[0] && d[1] == dimensions_[1]); } if (std::vector h; spec.get("halo", h)) { ASSERT(h.size() == 4); ASSERT(h[0] == halo_[0] && h[1] == halo_[1] && h[2] == halo_[2] && h[3] == halo_[3]); } if (std::vector p; spec.get("pivot", p)) { ASSERT(p.size() == 2); ASSERT(types::is_approximately_equal(p[0], pivot_[0])); ASSERT(types::is_approximately_equal(p[1], pivot_[1])); } auto n = static_cast(dimensions_[0] * dimensions_[1]); ASSERT(n > 0); ASSERT(n == longitudes_.size()); ASSERT(n == latitudes_.size()); ASSERT(n == flags_.size()); } size_t ORCA::ORCARecord::write(const PathName& p, const std::string& compression) { codec::RecordWriter record; codec::ArrayShape shape{static_cast(dimensions_[0]), static_cast(dimensions_[1])}; record.compression(compression); record.set("version", 0); record.set("dimensions", dimensions_); record.set("halo", halo_); record.set("pivot", pivot_); record.set("longitude", codec::ArrayReference(longitudes_.data(), shape)); record.set("latitude", codec::ArrayReference(latitudes_.data(), shape)); record.set("flags", codec::ArrayReference(flags_.data(), shape)); return record.write(p); } Grid::iterator ORCA::cbegin() const { return iterator{new geo::iterator::Unstructured( *this, 0, std::make_shared(record_.longitudes_, record_.latitudes_))}; } Grid::iterator ORCA::cend() const { return iterator{new geo::iterator::Unstructured(*this)}; } Grid::uid_type ORCA::calculate_uid() const { return record_.calculate_uid(arrangement_); } Point ORCA::first_point() const { ASSERT(!empty()); return PointLonLat{record_.longitudes_.front(), record_.latitudes_.front()}; } Point ORCA::last_point() const { ASSERT(!empty()); return PointLonLat{record_.longitudes_.back(), record_.latitudes_.back()}; } std::vector ORCA::to_points() const { std::vector p; p.reserve(size()); for (size_t i = 0; i < size(); ++i) { p.emplace_back(PointLonLat{record_.longitudes_[i], record_.latitudes_[i]}); } return p; } std::pair, std::vector> ORCA::to_latlons() const { return {record_.latitudes_, record_.longitudes_}; } const Grid::order_type& ORCA::order() const { NOTIMP; } Grid::renumber_type ORCA::reorder(const order_type& to) const { NOTIMP; } Grid::Spec* ORCA::spec_from_uid(const uid_type& uid) { return GridSpecByUID::instance().get(uid).spec(); } Arrangement ORCA::arrangement_from_string(const std::string& str) { return str == "F" ? Arrangement::ORCA_F : str == "T" ? Arrangement::ORCA_T : str == "U" ? Arrangement::ORCA_U : str == "V" ? Arrangement::ORCA_V : str == "W" ? Arrangement::ORCA_W : throw SeriousBug("ORCA: unsupported arrangement '" + str + "'"); } std::string ORCA::arrangement_to_string(Arrangement a) { return a == Arrangement::ORCA_F ? "F" : a == Arrangement::ORCA_T ? "T" : a == Arrangement::ORCA_U ? "U" : a == Arrangement::ORCA_V ? "V" : a == Arrangement::ORCA_W ? "W" : throw SeriousBug("ORCA: unsupported arrangement '" + std::to_string(a) + "'", Here()); } void ORCA::fill_spec(spec::Custom& custom) const { auto grid = name_ + "_" + arrangement_to_string(arrangement_); custom.set("grid", grid); if (auto _uid = uid(); !GridSpecByName::instance().exists(grid) || GridSpecByName::instance().match(grid).spec(grid)->get_string("orca_uid") != _uid) { custom.set("uid", _uid); } } const std::string& ORCA::type() const { static const std::string type{"ORCA"}; return type; } Grid::BoundingBox* ORCA::calculate_bbox() const { return new BoundingBox; } static const GridRegisterType GRIDTYPE("ORCA"); } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/grid/Reduced.cc0000664000175000017500000000227615161702250020260 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/Reduced.h" #include "eckit/geo/Exceptions.h" namespace eckit::geo::grid { Point Reduced::first_point() const { ASSERT(!empty()); return PointLonLat{longitudes(0).front(), latitudes().front()}; } Point Reduced::last_point() const { ASSERT(!empty()); auto j = static_cast(ny()) - 1; ASSERT(0 <= j); return PointLonLat{longitudes(j).back(), latitudes().back()}; } const std::vector& Reduced::nxacc() const { if (nxacc_.empty()) { nxacc_.resize(1 + ny()); nxacc_.front() = 0; size_t j = 0; for (auto a = nxacc_.begin(), b = a + 1; b != nxacc_.end(); ++j, ++a, ++b) { *b = *a + nx(j); } ASSERT(nxacc_.back() == size()); } return nxacc_; } } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/grid/Reduced.h0000664000175000017500000000251715161702250020120 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Grid.h" #include "eckit/geo/area/BoundingBox.h" namespace eckit::geo::iterator { class Reduced; } namespace eckit::geo::grid { class Reduced : public Grid { public: // -- Methods size_t size() const override { return nxacc().back(); } std::vector shape() const override { return {size()}; } // -- Overridden methods [[nodiscard]] Point first_point() const override; [[nodiscard]] Point last_point() const override; // Methods virtual const std::vector& latitudes() const = 0; virtual std::vector longitudes(size_t) const = 0; protected: // -- Methods const std::vector& nxacc() const; virtual size_t nx(size_t) const = 0; virtual size_t ny() const = 0; private: // -- Members mutable std::vector nxacc_; // -- Friends friend class geo::iterator::Reduced; }; } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/grid/Regular.h0000664000175000017500000000320015161702250020134 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Grid.h" #include "eckit/geo/Range.h" #include "eckit/geo/order/Scan.h" namespace eckit::geo::iterator { class Regular; } // namespace eckit::geo::iterator namespace eckit::geo::grid { class Regular : public Grid { public: // -- Methods virtual double dx() const = 0; virtual double dy() const = 0; virtual size_t nx() const = 0; virtual size_t ny() const = 0; virtual const Range& x() const = 0; virtual const Range& y() const = 0; // -- Overridden methods iterator cbegin() const final; iterator cend() const final; size_t size() const final { return nx() * ny(); } std::vector shape() const final { return {ny(), nx()}; } const order_type& order() const final; renumber_type reorder(const order_type& to) const final; protected: // -- Constructors explicit Regular(const Spec&); explicit Regular(const Projection* = nullptr); // -- Overridden methods void fill_spec(spec::Custom&) const override; // -- Methods void order(const order_type& to); private: // -- Members order::Scan order_; // -- Friends friend class geo::iterator::Regular; }; } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/grid/Regular.cc0000664000175000017500000000265715161702250020311 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/grid/Regular.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/iterator/Regular.h" #include "eckit/geo/order/Scan.h" #include "eckit/spec/Custom.h" #include "eckit/types/Fraction.h" namespace eckit::geo::grid { double Regular::dx() const { return x().increment(); } double Regular::dy() const { return y().increment(); } Grid::iterator Regular::cbegin() const { return iterator{new geo::iterator::Regular(*this, 0)}; } Grid::iterator Regular::cend() const { return iterator{new geo::iterator::Regular(*this, size())}; } const Regular::order_type& Regular::order() const { return order_.order(); } Grid::renumber_type Regular::reorder(const order_type& to) const { return order_.reorder(to, nx(), ny()); } Regular::Regular(const Spec& spec) : order_(spec) {} Regular::Regular(const Projection*) {} void Regular::fill_spec(spec::Custom& custom) const { Grid::fill_spec(custom); } void Regular::order(const order_type& to) { order_.order(to); } } // namespace eckit::geo::grid eckit-2.0.7/src/eckit/geo/Search.h0000664000175000017500000000337715161702250017032 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/search/Tree.h" #include "eckit/spec/Custom.h" namespace eckit::geo { class Grid; } namespace eckit::geo { class Search { public: // -- Types using ValueType = search::Tree::Payload; using PointType = search::Tree::Point; using PointValueType = search::Tree::PointValueType; // -- Constructors explicit Search(const Grid&, const spec::Spec& = spec::Custom{}); Search(const Search&) = delete; Search(Search&&) = delete; // -- Operators void operator=(const Search&) = delete; void operator=(Search&&) = delete; // -- Methods /// Finds closest point to an input point PointValueType closestPoint(const PointType&) const; /// Finds closest N points to an input point void closestNPoints(const PointType&, size_t n, std::vector& closest) const; /// Finds closest points within a radius void closestWithinRadius(const PointType&, double radius, std::vector& closest) const; private: // -- Members std::unique_ptr tree_; // -- Methods void build(const Grid&); void print(std::ostream&) const; // -- Friends friend std::ostream& operator<<(std::ostream& out, const Search& p) { p.print(out); return out; } }; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/area/0000775000175000017500000000000015161702250016352 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/area/Polygon.h0000664000175000017500000000344515161702250020160 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Area.h" #include "eckit/geo/PointXY.h" #include "eckit/geo/polygon/Polygon.h" namespace eckit::geo::area { class Polygon : public Area, protected std::vector { public: // -- Types using container_type = vector; using value_type = container_type::value_type; // -- Constructors explicit Polygon(const Area::Spec&); Polygon(const Polygon& other) : Area(other), container_type(other) {} Polygon(Polygon&& other) : Area(other), container_type(other) {} Polygon(const container_type& cont) : vector(cont) {} Polygon(container_type&& cont) : vector(cont) {} // -- Destructor ~Polygon() override = default; // -- Operators Polygon& operator=(const Polygon& other) { container_type::operator=(other); return *this; } Polygon& operator=(Polygon&& other) { container_type::operator=(other); return *this; } // -- Methods using vector::empty; // -- Overridden methods const std::string& type() const override; bool intersects(BoundingBox&) const override; bool contains(const Point&) const override; double area() const override; // -- Class methods [[nodiscard]] static Polygon* make_from_spec(const Area::Spec&); private: void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::area eckit-2.0.7/src/eckit/geo/area/BoundingBoxXY.cc0000664000175000017500000000531315161702250021362 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/area/BoundingBoxXY.h" #include #include #include "eckit/geo/Exceptions.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::area { static const AreaRegisterType AREATYPE("bounding_box_xy"); BoundingBoxXY::BoundingBoxXY(const Spec& spec) : BoundingBoxXY(*std::unique_ptr(make_from_spec(spec))) {} BoundingBoxXY::BoundingBoxXY(value_type min_x, value_type min_y, value_type max_x, value_type max_y) : array{min_x, min_y, max_x, max_y} { ASSERT(min_x <= max_x); ASSERT(min_y <= max_y); } bool BoundingBoxXY::contains(const Point& p) const { const auto& q = std::get(p); return types::is_approximately_equal(min_x, q.X()) && types::is_approximately_equal(q.X(), max_x) && types::is_approximately_equal(min_y, q.Y()) && types::is_approximately_equal(q.Y(), max_y); } bool BoundingBoxXY::contains(const BoundingBoxXY& other) const { return contains(PointXY{other.min_x, other.min_y}) && contains(PointXY{other.max_x, other.max_y}); } bool BoundingBoxXY::empty() const { return types::is_approximately_equal(min_x, max_x) || types::is_approximately_equal(min_y, max_y); } const std::string& BoundingBoxXY::type() const { static const std::string type{"bounding_box_xy"}; return type; } void BoundingBoxXY::fill_spec(spec::Custom& custom) const { custom.set("type", type()); custom.set(type(), std::vector{min_x, min_y, max_x, max_y}); } BoundingBoxXY* BoundingBoxXY::make_from_spec(const Spec& spec) { if (std::vector area; spec.get("bounding_box_xy", area)) { ASSERT_MSG(area.size() == 4, "BoundingBox: 'bounding_box_xy' expected list of size 4"); return new BoundingBoxXY{area[0], area[1], area[2], area[3]}; } if (std::vector area(4); spec.get("min_x", area[0]) && spec.get("min_y", area[1]) && spec.get("max_x", area[2]) && spec.get("max_y", area[3])) { return new BoundingBoxXY{area[0], area[1], area[2], area[3]}; } throw exception::SpecError("BoundingBoxXY: cannot build, expecting bounding_box_xy: [min_x, min_y, max_x, max_y]", Here()); } } // namespace eckit::geo::area eckit-2.0.7/src/eckit/geo/area/None.h0000664000175000017500000000150615161702250017424 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/Area.h" namespace eckit::geo::area { class None final : public Area { public: // -- Constructors None() = default; // -- Overriden methods const std::string& type() const override; bool intersects(area::BoundingBox&) const override { return false; } private: // -- Overriden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::area eckit-2.0.7/src/eckit/geo/area/Polygon.cc0000664000175000017500000000363015161702250020312 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/area/Polygon.h" #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/spec/Custom.h" namespace eckit::geo::area { static const AreaRegisterType AREATYPE("polygon"); Polygon::Polygon(const Area::Spec& spec) : Polygon(*std::unique_ptr(make_from_spec(spec))) {} bool Polygon::intersects(BoundingBox&) const { NOTIMP; } bool Polygon::contains(const Point& p) const { // NOTE This implementation follows that polygon rings alternatingly include/exclude the point (typical in GeoJSON); // In shapefile polygons (not here) the first ring tests inclusion and subsequent rings test exclusion, but point // ordering is assumed to be respected (ccw for inclusion, cw for exclusion) which isn't always the case const auto& q = std::get(p); auto c = std::count_if(begin(), end(), [&q](const auto& poly) { return poly.contains(q); }); return c % 2 != 0; } double Polygon::area() const { // NOTE this is "signed area", meaning it respects the point ordering return std::accumulate(begin(), end(), 0., [](auto a, const auto& poly) { return a + poly.area(); }); } const std::string& Polygon::type() const { static const std::string type{"polygon"}; return type; } void Polygon::fill_spec(spec::Custom& custom) const { custom.set("type", type()); custom.set(type(), "NOTIMP"); } Polygon* Polygon::make_from_spec(const Area::Spec& spec) { NOTIMP; } } // namespace eckit::geo::area eckit-2.0.7/src/eckit/geo/area/Library.h0000664000175000017500000000257315161702250020136 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include namespace eckit { namespace geo { class Area; } namespace spec { class Custom; class Spec; } // namespace spec } // namespace eckit namespace eckit::geo::area { class Library { public: // -- Constructors Library(const Library&) = delete; Library(Library&&) = delete; // -- Destructor virtual ~Library() = default; // -- Operators void operator=(const Library&) = delete; void operator=(Library&&) = delete; // -- Methods virtual std::ostream& list(std::ostream&) const = 0; virtual size_t size() const = 0; [[nodiscard]] virtual Area* make_area(size_t) const = 0; [[nodiscard]] virtual Area* make_area_from_name(const std::string&) const = 0; protected: // -- Constructors Library() = default; // -- Methods virtual void fill_spec(spec::Custom&) const = 0; }; } // namespace eckit::geo::area eckit-2.0.7/src/eckit/geo/area/BoundingBox.cc0000664000175000017500000004665415161702250021116 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/area/BoundingBox.h" #include #include #include #include #include #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/Projection.h" #include "eckit/geo/figure/Earth.h" #include "eckit/geo/figure/Sphere.h" #include "eckit/geo/projection/Composer.h" #include "eckit/geo/projection/EquidistantCylindrical.h" #include "eckit/geo/projection/Reverse.h" #include "eckit/geo/projection/Rotation.h" #include "eckit/geo/util.h" #include "eckit/spec/Custom.h" #include "eckit/types/FloatCompare.h" namespace eckit::geo::area { static const AreaRegisterType AREATYPE("bounding_box"); namespace { PointLonLat longitude_in_range(double reference, const PointLonLat& p) { // keep longitude difference (to reference) range below +-180 degree auto lon = p.lon(); while (lon > reference + 180.) { lon -= 360.; } while (lon <= reference - 180.) { lon += 360.; } return {lon, p.lat()}; } struct BoundLonLat { BoundLonLat(PointLonLat min, PointLonLat max) : min_(min), max_(max) {} explicit operator BoundingBox() const { // MIR-661 account for "excessive" bounds auto max_lon = std::min(max_.lon(), min_.lon() + PointLonLat::FULL_ANGLE); return {max_.lat(), min_.lon(), min_.lat(), max_lon}; } void extend(PointLonLat p, PointLonLat eps) { ASSERT(0. <= eps.lon() && 0. <= eps.lat()); auto sub = p - eps; auto add = p + eps; min_ = first_ ? sub : PointLonLat::componentsMin(min_, sub); max_ = first_ ? add : PointLonLat::componentsMax(max_, add); first_ = false; min_ = {min_.lon(), std::max(min_.lat(), -90.)}; max_ = {std::min(max_.lon(), min_.lon() + 360.), std::min(max_.lat(), 90.)}; ASSERT(min_.lon() <= max_.lon() && min_.lat() <= max_.lat()); includesSouthPole(types::is_approximately_equal(min_.lat(), -90.)); includesNorthPole(types::is_approximately_equal(max_.lat(), 90.)); crossesDateLine(types::is_approximately_equal(max_.lon() - min_.lon(), 360.)); } bool crossesDateLine(bool yes) { if ((crossesDateLine_ = crossesDateLine_ || yes)) { max_ = {min_.lon() + 360., max_.lat()}; } return crossesDateLine_; } bool includesNorthPole(bool yes) { if ((includesNorthPole_ = includesNorthPole_ || yes)) { max_ = {max_.lon(), 90.}; } crossesDateLine(includesNorthPole_); return includesNorthPole_; } bool includesSouthPole(bool yes) { if ((includesSouthPole_ = includesSouthPole_ || yes)) { min_ = {min_.lon(), -90.}; } crossesDateLine(includesSouthPole_); return includesSouthPole_; } bool crossesDateLine() const { return crossesDateLine_; } bool includesNorthPole() const { return includesNorthPole_; } bool includesSouthPole() const { return includesSouthPole_; } private: PointLonLat min_; PointLonLat max_; bool crossesDateLine_ = false; bool includesNorthPole_ = false; bool includesSouthPole_ = false; bool first_ = true; }; struct Derivate { Derivate(const Projection& p, PointXY A, PointXY B, double h, double refLongitude = 0.) : projection_(p), H_{PointXY::normalize(B - A) * h}, invnH_(1. / PointXY::norm(H_)), refLongitude_(refLongitude) {} virtual ~Derivate() = default; Derivate(const Derivate&) = delete; Derivate(Derivate&&) = delete; void operator=(const Derivate&) = delete; void operator=(Derivate&&) = delete; virtual PointLonLat d(PointXY) const = 0; PointLonLat f(const PointXY& p) const { return longitude_in_range(refLongitude_, std::get(projection_.inv(p))); } inline const PointXY& H() const { return H_; } inline double invnH() const { return invnH_; } private: const Projection& projection_; const PointXY H_; const double invnH_; const double refLongitude_; }; struct DerivateForwards final : Derivate { using Derivate::Derivate; PointLonLat d(PointXY P) const override { return (f(P + H()) - f(P)) * invnH(); } }; struct DerivateBackwards final : Derivate { using Derivate::Derivate; PointLonLat d(PointXY P) const override { return (f(P) - f(P - H())) * invnH(); } }; struct DerivateCentral final : Derivate { DerivateCentral(const Projection& p, PointXY A, PointXY B, double h, double refLongitude) : Derivate(p, A, B, h, refLongitude), H2_{H() * 0.5} {} const PointXY H2_; PointLonLat d(PointXY P) const override { return (f(P + H2_) - f(P - H2_)) * invnH(); } }; struct DerivateFactory { static const Derivate* build(const std::string& type, const Projection& p, PointXY A, PointXY B, double h, double refLongitude = 0.) { ASSERT(0. < h); if (A.distance2(B) < h * h) { struct DerivateDegenerate final : Derivate { using Derivate::Derivate; PointLonLat d(PointXY) const override { return {99, 99}; } // FIXME }; return new DerivateDegenerate(p, A, B, h, refLongitude); } return type == "forwards" ? static_cast(new DerivateForwards(p, A, B, h, refLongitude)) : type == "backwards" ? static_cast(new DerivateBackwards(p, A, B, h, refLongitude)) : type == "central" ? static_cast(new DerivateCentral(p, A, B, h, refLongitude)) : throw BadValue("DerivateFactory: unknown method", Here()); } static void list(std::ostream& out) { return instance().list_(out); } private: static DerivateFactory& instance() { static DerivateFactory obj; return obj; } // This is 'const' as Grid should always be immutable const Derivate* build_(const std::string& type, const Projection& p, PointXY A, PointXY B, double h, double refLongitude) const; void list_(std::ostream& out) const { out << "forwards, backwards, central" << std::endl; } }; [[nodiscard]] std::unique_ptr make_bounding_box(PointXY min, PointXY max, const Projection& projection, double precision_ll, double precision_xy) { using types::is_strictly_greater; // 0. setup // use central longitude as absolute reference (keep points within +-180 longitude range) const PointXY centre_xy{(min.X() + max.X()) / 2., (min.Y() + max.Y()) / 2.}; const auto centre_ll = std::get(projection.inv(centre_xy)); // asserts fwd(PointLonLat) -> PointXY const auto centre_lon = centre_ll.lon(); const std::string derivative_type = "central"; const double h_ll = precision_ll; const double h = precision_xy; constexpr size_t Niter = 100; // 1. determine box from projected corners struct : public std::pair { using pair::pair; bool contains(const PointXY& P) const { return (first.X() < P.X() && P.X() < second.X()) && (first.Y() < P.Y() && P.Y() < second.Y()); } } rect(min, max); const std::pair segments[] = {{{min.X(), max.Y()}, {max.X(), max.Y()}}, {{max.X(), max.Y()}, {max.X(), min.Y()}}, {{max.X(), min.Y()}, {min.X(), min.Y()}}, {{min.X(), min.Y()}, {min.X(), max.Y()}}}; BoundLonLat bounds(centre_ll, centre_ll); for (const auto& [A, dummy] : segments) { auto q = longitude_in_range(centre_lon, std::get(projection.inv(A))); bounds.extend(q, PointLonLat{h_ll, h_ll}); } // 2. locate latitude extrema by checking if poles are included (in the un-projected frame) and if not, find extrema // not at the corners by refining iteratively if (!bounds.includesNorthPole()) { bounds.includesNorthPole(rect.contains(std::get(projection.fwd(PointLonLat{0., 90. - h_ll})))); } if (!bounds.includesSouthPole()) { bounds.includesSouthPole(rect.contains(std::get(projection.fwd(PointLonLat{0., -90. + h_ll})))); } for (auto [A, B] : segments) { if (!bounds.includesNorthPole() || !bounds.includesSouthPole()) { std::unique_ptr derivate( DerivateFactory::build(derivative_type, projection, A, B, h, centre_lon)); double dAdy = derivate->d(A).lat(); double dBdy = derivate->d(B).lat(); if (!is_strictly_greater(dAdy * dBdy, 0.)) { PointLonLat H{0, h_ll}; for (size_t cnt = 0; cnt < Niter; ++cnt) { PointXY M = PointXY::middle(A, B); double dMdy = derivate->d(M).lat(); if (is_strictly_greater(dAdy * dMdy, 0.)) { A = M; dAdy = dMdy; } else if (is_strictly_greater(dBdy * dMdy, 0.)) { B = M; dBdy = dMdy; } else { break; } } // update extrema, extended by 'a small amount' (arbitrary) bounds.extend(std::get(projection.inv(PointXY::middle(A, B))), H); } } } // 3. locate longitude extrema not at the corners by refining iteratively for (auto [A, B] : segments) { if (!bounds.crossesDateLine()) { std::unique_ptr derivate( DerivateFactory::build(derivative_type, projection, A, B, h, centre_lon)); double dAdx = derivate->d(A).lon(); double dBdx = derivate->d(B).lon(); if (!is_strictly_greater(dAdx * dBdx, 0.)) { PointLonLat H{h_ll, 0}; for (size_t cnt = 0; cnt < Niter; ++cnt) { PointXY M = PointXY::middle(A, B); double dMdx = derivate->d(M).lon(); if (is_strictly_greater(dAdx * dMdx, 0.)) { A = M; dAdx = dMdx; } else if (is_strictly_greater(dBdx * dMdx, 0.)) { B = M; dBdx = dMdx; } else { break; } } // update extrema, extended by 'a small amount' (arbitrary) bounds.extend(std::get(projection.inv(PointXY::middle(A, B))), H); } } } // 4. return bounding box return std::make_unique(bounds); } } // namespace static inline bool is_approximately_equal(BoundingBox::value_type a, BoundingBox::value_type b) { return types::is_approximately_equal(a, b, PointLonLat::EPS); } std::unique_ptr BoundingBox::make_global_prime() { return std::make_unique(PointLonLat::RIGHT_ANGLE, 0., -PointLonLat::RIGHT_ANGLE, PointLonLat::FULL_ANGLE); } std::unique_ptr BoundingBox::make_global_antiprime() { return std::make_unique(PointLonLat::RIGHT_ANGLE, -PointLonLat::FLAT_ANGLE, -PointLonLat::RIGHT_ANGLE, PointLonLat::FLAT_ANGLE); } void BoundingBox::fill_spec(spec::Custom& custom) const { custom.set("area", std::vector{north(), west(), south(), east()}); } std::unique_ptr BoundingBox::make_from_spec(const Spec& spec) { auto [n, w, s, e] = bounding_box_default().deconstruct(); if (std::vector area{n, w, s, e}; spec.get("area", area)) { ASSERT_MSG(area.size() == 4, "BoundingBox: 'area' expected list of size 4"); return make_from_area(area[0], area[1], area[2], area[3]); } spec.get("north", n); spec.get("south", s); if (spec.get("west", w) && !spec.has("east")) { e = w + PointLonLat::FULL_ANGLE; } if (spec.get("east", e) && !spec.has("west")) { w = e - PointLonLat::FULL_ANGLE; } return make_from_area(n, w, s, e); } std::unique_ptr BoundingBox::make_from_projection(PointXY min, PointXY max, const Projection& projection) { constexpr double precision_ll = 0.5e-6; // to microdegrees constexpr double precision_xy = 0.5e-1; // to decimeters return make_bounding_box(min, max, projection, precision_ll, precision_xy); } std::unique_ptr BoundingBox::make_from_projection(PointLonLat min, PointLonLat max, const projection::Rotation& rotation) { projection::Composer projection{new projection::Reverse(rotation.spec()), new projection::EquidistantCylindrical}; auto after = make_from_projection(PointXY{min.lon(), min.lat()}, PointXY{max.lon(), max.lat()}, projection); if (after->periodic()) { return std::make_unique(after->north(), 0, after->south(), PointLonLat::FULL_ANGLE); } return after; } std::unique_ptr BoundingBox::make_from_area(value_type n, value_type w, value_type s, value_type e) { // set latitudes inside usual range (not a normalisation like PointLonLat::make) if (n > NORTH_POLE.lat() || is_approximately_equal(n, NORTH_POLE.lat())) { n = NORTH_POLE.lat(); } if (s < SOUTH_POLE.lat() || is_approximately_equal(s, SOUTH_POLE.lat())) { s = SOUTH_POLE.lat(); } // normalise west in [min, min + 2 pi[, east in [west, west + 2 pi[ const auto min = -PointLonLat::FLAT_ANGLE; const auto same = is_approximately_equal(w, e); w = is_approximately_equal(w, min) || is_approximately_equal(w, min + PointLonLat::FULL_ANGLE) ? min : PointLonLat::normalise_angle_to_minimum(w, min); auto a = PointLonLat::normalise_angle_to_minimum(e, w); e = same ? w : is_approximately_equal(w, a) ? (w + PointLonLat::FULL_ANGLE) : a; return std::make_unique(n, w, s, e); } BoundingBox::BoundingBox(const Spec& spec) : BoundingBox(*make_from_spec(spec)) {} BoundingBox::BoundingBox(value_type n, value_type w, value_type s, value_type e) : array{n, w, s, e} { auto is_approx_ge = [](value_type a, value_type b) { return is_approximately_equal(a, b) || a > b; }; // normalise east in [west, west + 2 pi[ auto a = PointLonLat::normalise_angle_to_minimum(e, w); operator[](3) = is_approximately_equal(w, e) ? w : is_approximately_equal(w, a) ? w + PointLonLat::FULL_ANGLE : is_approx_ge(e - w, PointLonLat::FULL_ANGLE) ? w + PointLonLat::FULL_ANGLE : a; ASSERT(south() <= north()); ASSERT(west() <= east()); } BoundingBox::BoundingBox() : BoundingBox(*make_global_prime()) {} bool BoundingBox::global() const { return periodic() && contains(NORTH_POLE) && contains(SOUTH_POLE); } bool BoundingBox::periodic() const { return west() != east() && is_approximately_equal(west(), PointLonLat::normalise_angle_to_minimum(east(), west())); } bool BoundingBox::contains(const Point& p) const { const auto& q = std::get(p); // NOTE: latitudes < -90 or > 90 are not considered if (is_approximately_equal(q.lat(), NORTH_POLE.lat())) { return is_approximately_equal(q.lat(), north()); } if (is_approximately_equal(q.lat(), SOUTH_POLE.lat())) { return is_approximately_equal(q.lat(), south()); } if ((south() < q.lat() && q.lat() < north()) || is_approximately_equal(q.lat(), north()) || is_approximately_equal(q.lat(), south())) { return PointLonLat::normalise_angle_to_minimum(q.lon(), west()) <= east(); } return false; } const BoundingBox& BoundingBox::bounding_box_default() { static const BoundingBox bbox; return bbox; } bool BoundingBox::contains(const BoundingBox& other) const { if (other.empty()) { return contains(PointLonLat{other.south(), other.west()}); } // check for West/East range (if non-periodic), then other's corners if (east() - west() < other.east() - other.west() || east() < PointLonLat::normalise_angle_to_minimum(other.east(), west())) { return false; } return contains(PointLonLat{other.north(), other.west()}) && contains(PointLonLat{other.north(), other.east()}) && contains(PointLonLat{other.south(), other.west()}) && contains(PointLonLat{other.south(), other.east()}); } bool BoundingBox::intersects(BoundingBox& other) const { auto n = std::min(north(), other.north()); auto s = std::max(south(), other.south()); bool intersectsSN = s <= n; if (!intersectsSN) { n = s; } if (periodic() && other.periodic()) { other = {n, other.west(), s, other.east()}; return intersectsSN; } auto w = std::min(west(), other.west()); auto e = w; auto intersect = [](const BoundingBox& a, const BoundingBox& b, value_type& w, value_type& e) { bool p = a.periodic(); if (p || b.periodic()) { w = (p ? b : a).west(); e = (p ? b : a).east(); return true; } auto ref = PointLonLat::normalise_angle_to_minimum(b.west(), a.west()); auto w_ = std::max(a.west(), ref); auto e_ = std::min(a.east(), PointLonLat::normalise_angle_to_minimum(b.east(), ref)); if (w_ <= e_) { w = w_; e = e_; return true; } return false; }; bool intersectsWE = west() <= other.west() ? intersect(*this, other, w, e) || intersect(other, *this, w, e) : intersect(other, *this, w, e) || intersect(*this, other, w, e); ASSERT_MSG(w <= e, "BoundingBox::intersects: longitude range"); other = {n, w, s, e}; return intersectsSN && intersectsWE; } bool BoundingBox::empty() const { return is_approximately_equal(north(), south()) || is_approximately_equal(west(), east()); } double BoundingBox::area() const { return figure::Earth::_area(*this); } const std::string& BoundingBox::type() const { static const std::string type{"bounding-box"}; return type; } bool bounding_box_equal(const BoundingBox& a, const BoundingBox& b) { const auto c = BoundingBox::make_from_area(a.north(), a.west(), a.south(), a.east()); ASSERT(c); const auto d = BoundingBox::make_from_area(b.north(), b.west(), b.south(), b.east()); ASSERT(c); return is_approximately_equal(c->north(), d->north()) && is_approximately_equal(c->south(), d->south()) && is_approximately_equal(c->west(), d->west()) && is_approximately_equal(c->east(), d->east()); } } // namespace eckit::geo::area eckit-2.0.7/src/eckit/geo/area/library/0000775000175000017500000000000015161702250020016 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/area/library/GeoJSON.h0000664000175000017500000000376415161702250021405 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/filesystem/PathName.h" #include "eckit/geo/area/Library.h" #include "eckit/geo/polygon/Polygon.h" #include "eckit/value/Value.h" namespace eckit::geo { class Area; } namespace eckit::geo::area::library { class GeoJSON : public Library { public: // -- Types /* * Note: * - geo::polygon::Polygon is a list of (geo::PointLonLat) points * - GeoJSON "Polygon" is a list of (geo::polygon::Polygon) polygon (eg. "polygon"+"hole"+"hole"+...) * - GeoJSON "MultiPolygon" is a list of GeoJSON "Polygon" */ using Polygons = std::vector; // -- Constructors explicit GeoJSON(const spec::Spec&); explicit GeoJSON(const PathName&, const std::string& name = ""); // -- Overridden methods std::ostream& list(std::ostream&) const override; size_t size() const override { return polygons_.size(); } [[nodiscard]] Area* make_area(size_t) const override; [[nodiscard]] Area* make_area_from_name(const std::string&) const override; // -- Class methods [[nodiscard]] static GeoJSON* make_from_json_string(const std::string& json, const std::string& name = ""); private: // -- Constructors explicit GeoJSON(const Value&, const std::string& file, const std::string& name); // -- Members const PathName file_; const std::string name_; std::map to_index; std::vector polygons_; // -- Overridden methods void fill_spec(eckit::spec::Custom&) const override; }; } // namespace eckit::geo::area::library eckit-2.0.7/src/eckit/geo/area/library/GeoJSON.cc0000664000175000017500000001101015161702250021522 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/area/library/GeoJSON.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/PointLonLat.h" #include "eckit/geo/area/Polygon.h" #include "eckit/log/JSON.h" #include "eckit/parser/JSONParser.h" #include "eckit/spec/Custom.h" namespace eckit::geo::area::library { namespace { ValueList list(const Value& j) { ASSERT(j.isList()); return j.as(); } polygon::Polygon polygon(const Value& j) { auto c = list(j); polygon::Polygon::container_type p; p.reserve(c.size()); for (auto& l : c) { auto lonlat = list(l); ASSERT(lonlat.size() == 2); ASSERT(lonlat[0].isDouble() || lonlat[0].isNumber()); ASSERT(lonlat[1].isDouble() || lonlat[1].isNumber()); p.emplace_back(lonlat[0].as(), lonlat[1].as()); } return polygon::Polygon(p); } void build_polygons(const Value& j, std::vector& all) { if (j.isList()) { for (const auto& l : list(j)) { build_polygons(l, all); } return; } if (j.isMap()) { if (j["type"].isString() && j["type"].as() == "Polygon") { // "coordinates" is (a list) of (a list) of coordinates (lists of size 2) bool first = true; for (auto& lc : list(j["coordinates"])) { if (first) { first = false; all.emplace_back(); } all.back().emplace_back(polygon(lc)); } } if (j["type"].isString() && j["type"].as() == "MultiPolygon") { // "coordinates" is (a list) of (a list) of (a list) of coordinates (lists of size 2) bool first = true; for (auto& llc : list(j["coordinates"])) { for (auto& lc : list(llc)) { if (first) { first = false; all.emplace_back(); } all.back().emplace_back(polygon(lc)); } } } for (const auto& k : ValueMap(j)) { build_polygons(k.second, all); } return; } } } // namespace GeoJSON::GeoJSON(const spec::Spec& spec) : GeoJSON(PathName(spec.get_string("file")), spec.get_string("name", "")) {} GeoJSON::GeoJSON(const PathName& file, const std::string& name) : GeoJSON(JSONParser::decodeFile(file), file, name) {} GeoJSON::GeoJSON(const Value& json, const std::string& file, const std::string& name) : file_(file), name_(name) { build_polygons(json, polygons_); if (!name_.empty()) { ASSERT(json.isMap() && json["features"].isList()); int _index = 0; for (const auto& feature : json["features"].as()) { ASSERT(feature.isMap() && feature["properties"].isMap() && feature["properties"][name].isString()); auto _name = feature["properties"][name].as(); ASSERT(to_index.emplace(_name, _index++).second); } } } void GeoJSON::fill_spec(eckit::spec::Custom& custom) const { custom.set("type", "geojson"); custom.set("file", file_.asString()); if (!name_.empty()) { custom.set("name", name_); } } GeoJSON* GeoJSON::make_from_json_string(const std::string& json, const std::string& name) { return new GeoJSON(JSONParser::decodeString(json), "", name); } Area* GeoJSON::make_area(size_t index) const { return new area::Polygon(polygons_[index]); } Area* GeoJSON::make_area_from_name(const std::string& name) const { ASSERT(!name_.empty() && !name.empty()); return make_area(to_index.at(name)); } std::ostream& GeoJSON::list(std::ostream& out) const { JSON j(out); j.startObject(); j << "type" << "geojson"; if (!file_.asString().empty()) { j << "file" << file_; } j << "size" << size(); if (!name_.empty()) { j << "name" << name_; j << "areas"; j.startList(); for (const auto& [key, val_] : to_index) { j << key; } j.endList(); } j.endObject(); return out; } } // namespace eckit::geo::area::library eckit-2.0.7/src/eckit/geo/area/library/Shapefile.cc0000664000175000017500000001623415161702250022233 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/area/library/Shapefile.h" #include #include #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/geo/area/Polygon.h" #include "eckit/geo/cache/Download.h" #include "eckit/geo/cache/Unzip.h" #include "eckit/geo/polygon/Polygon.h" #include "eckit/log/JSON.h" #include "eckit/log/Log.h" #include "eckit/spec/Custom.h" namespace eckit::geo::area::library { namespace { PathName path_shp(const PathName& file) { if (file.extension() == ".zip") { // check for exactly one .shp file in the zip if (auto list = cache::Unzip::list(file, true); std::count_if(list.begin(), list.end(), [](const auto& what) { return PathName(what).extension() == ".shp"; }) != 1) { throw ReadError("Shapefile: found none/more than one .shp file(s) in '" + file + "', expecting one", Here()); } static cache::Unzip unzip(LibEcKitGeo::cacheDir() + "/shapefile"); auto dir = unzip.to_cached_path(file); std::vector files; std::vector dirs; dir.children(files, dirs); files.erase( std::remove_if(files.begin(), files.end(), [](const auto& file) { return file.extension() != ".shp"; }), files.end()); if (files.size() != 1) { throw ReadError("Shapefile: found none/more than one .shp file(s) in '" + file + "', expecting one", Here()); } return files.front(); } return file; } PathName path_dbf(const PathName& file, const PathName& shp) { if (file.asString().empty()) { auto str = shp.asString(); return str.substr(0, str.length() - 4) + ".dbf"; } return file; } } // namespace Shapefile::Shapefile(const spec::Spec& spec) : Shapefile(path_shp(spec.get_string(spec.has("shp") ? "shp" : "file")), spec.has("dbf") ? spec.get_string("dbf") : "", spec.has("name_field") ? spec.get_string("name_field") : "") {} Shapefile::Shapefile(const PathName& file) : Shapefile(file, "") {} Shapefile::Shapefile(const PathName& shp, const PathName& dbf, const std::string& name) : shpPath_(path_shp(shp)), dbfPath_(path_dbf(dbf, shpPath_)), nEntities_(0) { Log::debug() << "eckit::geo::area::library::Shapefile(shp='" << shpPath_.realName() << "',dbf='" << dbfPath_.realName() << "',name='" << name << "')" << std::endl; if ((shp_ = SHPOpen(shpPath_.localPath(), "rb")) == nullptr) { throw CantOpenFile(shpPath_ + " (as .shp)", Here()); } int type = 0; SHPGetInfo(shp_, &nEntities_, &type, nullptr, nullptr); if (type != SHPT_ARC && type != SHPT_POLYGON) { throw ReadError("Shapefile: unsupported shape type", Here()); } if (!name.empty()) { DBFInfo* dbf = DBFOpen(dbfPath_.localPath(), "rb"); if (dbf == nullptr) { throw CantOpenFile(shpPath_ + " (as .dbf)", Here()); } // find named field index std::map to_index; char fieldName[12]; for (int i = 0, n = DBFGetFieldCount(dbf); i < n; ++i) { if (DBFGetNativeFieldType(dbf, i) == 'C') { DBFGetFieldInfo(dbf, i, fieldName, nullptr, nullptr); if (name == fieldName) { name_ = name; // map entities names (they have to be unique) for (int e = 0; e < nEntities_; ++e) { ASSERT(to_entity.emplace(DBFReadStringAttribute(dbf, e, i), e).second); } break; } } } DBFClose(dbf); if (name_.empty()) { throw ReadError("Shapefile: field '" + name + "' not found in '" + dbfPath_ + "'", Here()); } } } Shapefile::~Shapefile() { SHPClose(shp_); } Shapefile* Shapefile::make_from_url(const std::string& url) { static cache::Download download(LibEcKitGeo::cacheDir() + "/shapefile"); return new Shapefile(download.to_cached_path(url, cache::Download::url_file_basename(url, false), cache::Download::url_file_extension(url))); } void Shapefile::fill_spec(spec::Custom& custom) const { custom.set("type", "shapefile"); custom.set("shp", shpPath_); custom.set("shp", dbfPath_); if (!name_.empty()) { custom.set("field", name_); } } std::ostream& Shapefile::list(std::ostream& out) const { JSON j(out); j.startObject(); j << "type" << "shapefile"; j << "shp" << shpPath_; j << "dbf" << dbfPath_; j << "size" << size(); if (!name_.empty()) { j << "name" << name_; j << "areas"; j.startList(); for (const auto& [key, val_] : to_entity) { j << key; } j.endList(); } j.endObject(); return out; } Area* Shapefile::make_area_from_name(const std::string& name) const { ASSERT(!name_.empty() && !name.empty()); return make_area(to_entity.at(name)); } Area* Shapefile::make_area(size_t entity) const { struct Object : std::unique_ptr { explicit Object(SHPObject* ptr) : unique_ptr{ptr, SHPDestroyObject} { ASSERT(operator bool()); } }; Object obj(SHPReadObject(shp_, static_cast(entity))); ASSERT(obj); ASSERT(obj->nSHPType == SHPT_ARC || obj->nSHPType == SHPT_POLYGON); area::Polygon::container_type parts; for (int p = 0; p < obj->nParts; ++p) { auto start = obj->panPartStart[p]; auto end = (p == obj->nParts - 1) ? obj->nVertices : obj->panPartStart[p + 1]; if (start < end) { using points_type = polygon::Polygon::container_type; // only process closed loops points_type::value_type first(obj->padfX[start], obj->padfY[start]); points_type::value_type last(obj->padfX[end - 1], obj->padfY[end - 1]); if (points_equal(first, last)) { points_type pts; pts.reserve(end - start); for (auto j = start; j < end; ++j) { pts.emplace_back(obj->padfX[j], obj->padfY[j]); } parts.emplace_back(pts); } } } return new area::Polygon(parts); } #if 0 class AreaLibrary { public: void add_library(const std::string& lib, const Value& value) { std::unique_ptr spec(spec::Custom::make_from_value(value)); emplace(lib, ShapefileURL{LibEcKitGeo::url(spec->get_string("url"))}); } }; #endif } // namespace eckit::geo::area::library eckit-2.0.7/src/eckit/geo/area/library/Shapefile.h0000664000175000017500000000350615161702250022073 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/filesystem/PathName.h" #include "eckit/geo/area/Library.h" #include "shapefil.h" namespace eckit::geo { class Area; } namespace eckit::geo::area::library { class Shapefile : public Library { public: // -- Constructors explicit Shapefile(const spec::Spec&); explicit Shapefile(const PathName&); explicit Shapefile(const PathName& shp, const PathName& dbf, const std::string& name = ""); Shapefile(const Shapefile&) = delete; Shapefile(Shapefile&&) = delete; // -- Destructor ~Shapefile() override; // -- Operators void operator=(const Shapefile&) = delete; void operator=(Shapefile&&) = delete; // -- Overridden methods std::ostream& list(std::ostream&) const override; size_t size() const override { return static_cast(nEntities_); } [[nodiscard]] Area* make_area(size_t) const override; [[nodiscard]] Area* make_area_from_name(const std::string&) const override; // -- Class methods [[nodiscard]] static Shapefile* make_from_url(const std::string&); private: // -- Members const PathName shpPath_; const PathName dbfPath_; SHPInfo* shp_; std::string name_; int nEntities_; std::map to_entity; // -- Overridden methods void fill_spec(spec::Custom&) const override; }; } // namespace eckit::geo::area::library eckit-2.0.7/src/eckit/geo/area/BoundingBox.h0000664000175000017500000000664515161702250020754 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/Area.h" #include "eckit/geo/PointLonLat.h" #include "eckit/geo/PointXY.h" namespace eckit::geo { namespace area { class BoundingBox; bool bounding_box_equal(const BoundingBox&, const BoundingBox&); } // namespace area class Projection; namespace projection { class Rotation; } } // namespace eckit::geo namespace eckit::geo::area { class BoundingBox : public Area, protected std::array { public: // -- Types using container_type = array; using value_type = container_type::value_type; // -- Constructors explicit BoundingBox(const Spec&); BoundingBox(value_type north, value_type west, value_type south, value_type east); BoundingBox(); BoundingBox(const BoundingBox& other) : Area(other), container_type(other) {} BoundingBox(BoundingBox&& other) : Area(other), container_type(other) {} // -- Destructor ~BoundingBox() override = default; // -- Operators bool operator==(const BoundingBox& other) const { return bounding_box_equal(*this, other); } bool operator!=(const BoundingBox& other) const { return !operator==(other); } BoundingBox& operator=(const BoundingBox& other) { container_type::operator=(other); return *this; } BoundingBox& operator=(BoundingBox&& other) { container_type::operator=(other); return *this; } // -- Methods container_type deconstruct() const { return {north(), west(), south(), east()}; } bool global() const; bool periodic() const; bool contains(const BoundingBox&) const; bool empty() const; /// Surface area, assuming Earth [L^2] double area() const override; // -- Overridden methods const std::string& type() const override; void fill_spec(spec::Custom&) const override; bool intersects(BoundingBox&) const override; bool contains(const Point&) const override; // -- Class methods static const BoundingBox& bounding_box_default(); [[nodiscard]] static std::unique_ptr make_global_prime(); [[nodiscard]] static std::unique_ptr make_global_antiprime(); [[nodiscard]] static std::unique_ptr make_from_area(value_type n, value_type w, value_type s, value_type e); [[nodiscard]] static std::unique_ptr make_from_spec(const Spec&); [[nodiscard]] static std::unique_ptr make_from_projection(PointXY min, PointXY max, const Projection&); [[nodiscard]] static std::unique_ptr make_from_projection(PointLonLat min, PointLonLat max, const projection::Rotation&); // -- Members value_type north() const { return operator[](0); } value_type west() const { return operator[](1); } value_type south() const { return operator[](2); } value_type east() const { return operator[](3); } }; } // namespace eckit::geo::area eckit-2.0.7/src/eckit/geo/area/None.cc0000664000175000017500000000131715161702250017562 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/area/None.h" #include "eckit/spec/Custom.h" namespace eckit::geo::area { const std::string& None::type() const { static const std::string type{"none"}; return type; } void None::fill_spec(spec::Custom& custom) const { custom.set("type", type()); } } // namespace eckit::geo::area eckit-2.0.7/src/eckit/geo/area/Library.cc0000664000175000017500000000073615161702250020273 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/area/Library.h" namespace eckit::geo::area { // TODO } eckit-2.0.7/src/eckit/geo/area/BoundingBoxXY.h0000664000175000017500000000400515161702250021221 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/geo/Area.h" #include "eckit/geo/PointXY.h" namespace eckit::geo::area { class BoundingBoxXY : public Area, protected std::array { public: // -- Types using container_type = array; using value_type = container_type::value_type; // -- Constructors explicit BoundingBoxXY(const Spec&); BoundingBoxXY(value_type min_x, value_type min_y, value_type max_x, value_type max_y); BoundingBoxXY(const BoundingBoxXY& other) : Area(other), container_type(other) {} BoundingBoxXY(BoundingBoxXY&& other) : Area(other), container_type(other) {} // -- Destructor ~BoundingBoxXY() override = default; // -- Operators BoundingBoxXY& operator=(const BoundingBoxXY& other) { container_type::operator=(other); return *this; } BoundingBoxXY& operator=(BoundingBoxXY&& other) { container_type::operator=(other); return *this; } // -- Methods container_type deconstruct() const { return {min_x, min_y, max_x, max_y}; } bool contains(const Point&) const override; bool contains(const BoundingBoxXY&) const; bool empty() const; // -- Overridden methods const std::string& type() const override; void fill_spec(spec::Custom&) const override; // -- Class methods [[nodiscard]] static BoundingBoxXY* make_from_spec(const Spec&); // -- Members const value_type& min_x = operator[](0); const value_type& min_y = operator[](1); const value_type& max_x = operator[](2); const value_type& max_y = operator[](3); }; } // namespace eckit::geo::area eckit-2.0.7/src/eckit/geo/Exceptions.cc0000664000175000017500000000316415161702250020076 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/Exceptions.h" namespace eckit::geo::exception { AreaError::AreaError(const std::string& what, const CodeLocation& location) : Exception("AreaError: [" + what + "]", location) {} FigureError::FigureError(const std::string& what, const CodeLocation& location) : Exception("FigureError: [" + what + "]", location) {} GridError::GridError(const std::string& what, const CodeLocation& location) : Exception("GridError: [" + what + "]", location) {} GridUnknownError::GridUnknownError(const std::string& what, const CodeLocation& location) : Exception("GridUnknownError: [" + what + "]", location) {} OrderError::OrderError(const std::string& what, const CodeLocation& location) : Exception("OrderError: [" + what + "]", location) {} ProjectionError::ProjectionError(const std::string& what, const CodeLocation& location) : Exception("ProjectionError: [" + what + "]", location) {} RangeError::RangeError(const std::string& what, const CodeLocation& location) : Exception("RangeError: [" + what + "]", location) {} SearchError::SearchError(const std::string& what, const CodeLocation& location) : Exception("SearchError: [" + what + "]", location) {} } // namespace eckit::geo::exception eckit-2.0.7/src/eckit/geo/CMakeLists.txt0000664000175000017500000001352415161702250020207 0ustar alastairalastair list(APPEND eckit_geo_srcs Area.cc Area.h Arrangement.h Container.h Exceptions.cc Exceptions.h Figure.cc Figure.h GreatCircle.cc GreatCircle.h Grid.cc Grid.h Iterator.cc Iterator.h LibEcKitGeo.cc LibEcKitGeo.h Point.cc Point.h PointLonLat.cc PointLonLat.h PointLonLatR.cc PointLonLatR.h PointXY.cc PointXY.h PointXYZ.cc PointXYZ.h Projection.cc Projection.h Range.cc Range.h Search.cc Search.h Shape.cc Shape.h Trace.cc Trace.h area/BoundingBox.cc area/BoundingBox.h area/BoundingBoxXY.cc area/BoundingBoxXY.h area/Library.cc area/Library.h area/None.cc area/None.h area/Polygon.cc area/Polygon.h area/library/GeoJSON.cc area/library/GeoJSON.h cache/DiskCache.cc cache/DiskCache.h cache/Download.cc cache/Download.h cache/MemoryCache.cc cache/MemoryCache.h cache/Record.cc cache/Record.h container/PointsContainer.cc container/PointsContainer.h figure/Earth.cc figure/Earth.h figure/OblateSpheroid.cc figure/OblateSpheroid.h figure/OblateSpheroidT.h figure/Sphere.cc figure/Sphere.h figure/SphereT.h figure/UnitSphere.cc figure/UnitSphere.h grid/Reduced.cc grid/Reduced.h grid/Regular.cc grid/Regular.h grid/SphericalHarmonics.cc grid/SphericalHarmonics.h grid/Unstructured.cc grid/Unstructured.h grid/reduced/HEALPix.cc grid/reduced/HEALPix.h grid/reduced/ReducedGaussian.cc grid/reduced/ReducedGaussian.h grid/reduced/ReducedLonLat.cc grid/reduced/ReducedLonLat.h grid/regular/RegularGaussian.cc grid/regular/RegularGaussian.h grid/regular/RegularLL.cc grid/regular/RegularLL.h grid/regular/RegularXY.cc grid/regular/RegularXY.h grid/unstructured/UnstructuredGeneric.cc grid/unstructured/UnstructuredGeneric.h iterator/Reduced.cc iterator/Reduced.h iterator/Regular.cc iterator/Regular.h iterator/Unstructured.cc iterator/Unstructured.h order/HEALPix.cc order/HEALPix.h order/Scan.cc order/Scan.h polygon/Polygon.cc polygon/Polygon.h polygon/PolygonXY.cc polygon/PolygonXY.h projection/AlbersEqualArea.cc projection/AlbersEqualArea.h projection/Composer.cc projection/Composer.h projection/EquidistantCylindrical.cc projection/EquidistantCylindrical.h projection/LambertAzimuthalEqualArea.cc projection/LambertAzimuthalEqualArea.h projection/LambertConformalConic.cc projection/LambertConformalConic.h projection/LonLatToXYZ.cc projection/LonLatToXYZ.h projection/Mercator.cc projection/Mercator.h projection/None.cc projection/None.h projection/PolarStereographic.cc projection/PolarStereographic.h projection/Reverse.h projection/Rotation.cc projection/Rotation.h projection/SpaceView.cc projection/SpaceView.h projection/Stretch.cc projection/Stretch.h range/GaussianLatitude.cc range/GaussianLatitude.h range/Regular.cc range/Regular.h search/Tree.cc search/Tree.h search/TreeMapped.cc search/TreeMapped.h search/TreeMappedAnonymousMemory.cc search/TreeMappedAnonymousMemory.h search/TreeMappedFile.cc search/TreeMappedFile.h search/TreeMemory.cc search/TreeMemory.h share/Area.cc share/Area.h share/Grid.cc share/Grid.h share/Projection.cc share/Projection.h util.cc util.h util/arange.cc util/gaussian_latitudes.cc util/gaussian_quadrature_weights.cc util/hash_vector.cc util/linspace.cc util/monotonic_crop.cc util/mutex.h util/reduced_classical_pl.cc util/reduced_octahedral_pl.cc util/reverse.cc util/sincos.h ) set(eckit_geo_include_dirs $ $ ) set(eckit_geo_libs eckit_maths eckit_spec) set(eckit_GEO_SHARE_AREA "~eckit/share/eckit/geo/area.yaml") set(eckit_GEO_SHARE_GRID "~eckit/share/eckit/geo/grid.yaml") set(eckit_GEO_SHARE_PROJECTION "~eckit/share/eckit/geo/projection.yaml") if(eckit_HAVE_PROJ) list(APPEND eckit_geo_srcs projection/PROJ.cc projection/PROJ.h) list(APPEND eckit_geo_libs PROJ::proj) endif() if(ECKIT_GEO_CODEC_GRIDS) list(APPEND eckit_geo_srcs grid/unstructured/FESOM.cc grid/unstructured/FESOM.h grid/unstructured/ICON.cc grid/unstructured/ICON.h grid/ORCA.cc grid/ORCA.h ) list(APPEND eckit_GEO_SHARE_GRID "~eckit/share/eckit/geo/FESOM.yaml" "~eckit/share/eckit/geo/ICON-CH.yaml" "~eckit/share/eckit/geo/ICON-DWD.yaml" "~eckit/share/eckit/geo/ICON-MPIM.yaml" "~eckit/share/eckit/geo/ORCA.yaml" ) list(APPEND eckit_geo_libs eckit_codec) endif() if(eckit_HAVE_GEO_AREA_SHAPEFILE) list(APPEND eckit_geo_srcs area/library/Shapefile.cc area/library/Shapefile.h cache/Unzip.cc cache/Unzip.h ) list(APPEND eckit_geo_libs shapelib::shp libzip::zip) endif() string(REPLACE ";" ":" eckit_GEO_SHARE_GRID "${eckit_GEO_SHARE_GRID}") string(REPLACE ";" ":" eckit_GEO_SHARE_AREA "${eckit_GEO_SHARE_AREA}") string(REPLACE ";" ":" eckit_GEO_SHARE_PROJECTION "${eckit_GEO_SHARE_PROJECTION}") configure_file(eckit_geo_config.h.in eckit_geo_config.h @ONLY) list(APPEND eckit_geo_srcs ${CMAKE_CURRENT_BINARY_DIR}/eckit_geo_config.h) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/eckit_geo_config.h DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/geo ) ecbuild_add_library( TARGET eckit_geo TYPE SHARED INSTALL_HEADERS ALL HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/geo PUBLIC_LIBS ${eckit_geo_libs} PUBLIC_INCLUDES ${eckit_geo_include_dirs} SOURCES ${eckit_geo_srcs} ) eckit-2.0.7/src/eckit/geo/search/0000775000175000017500000000000015161702250016707 5ustar alastairalastaireckit-2.0.7/src/eckit/geo/search/TreeMappedAnonymousMemory.h0000664000175000017500000000145715161702250024217 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/geo/search/TreeMapped.h" namespace eckit::geo::search { class TreeMappedAnonymousMemory : public TreeMapped { bool ready() const override { return false; } void commit() override {} void print(std::ostream& out) const override { out << "TreeMappedAnonymousMemory[]"; } public: explicit TreeMappedAnonymousMemory(const Grid&); }; } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/search/TreeMapped.cc0000664000175000017500000000334115161702250021245 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/search/TreeMapped.h" namespace eckit::geo::search { void TreeMapped::build(std::vector& v) { tree_.build(v); } void TreeMapped::insert(const Tree::PointValueType& pt) { tree_.insert(pt); } void TreeMapped::statsPrint(std::ostream& out, bool pretty) { tree_.statsPrint(out, pretty); } void TreeMapped::statsReset() { tree_.statsReset(); } Tree::PointValueType TreeMapped::nearestNeighbour(const Tree::Point& pt) { const auto& nn = tree_.nearestNeighbour(pt).value(); return {nn.point(), nn.payload()}; } std::vector TreeMapped::kNearestNeighbours(const Tree::Point& pt, size_t k) { std::vector result; for (const auto& n : tree_.kNearestNeighbours(pt, k)) { result.emplace_back(PointValueType(n.point(), n.payload())); } return result; } std::vector TreeMapped::findInSphere(const Tree::Point& pt, double radius) { std::vector result; for (const auto& n : tree_.findInSphere(pt, radius)) { result.emplace_back(PointValueType(n.point(), n.payload())); } return result; } TreeMapped::TreeMapped(const Grid& r, const PathName& path) : Tree(r), umask_(0), path_(path), tree_(path, path.exists() ? 0 : itemCount(), 0) {} } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/search/Tree.cc0000664000175000017500000000735515161702250020127 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/search/Tree.h" #include #include #include "eckit/geo/Exceptions.h" #include "eckit/geo/Grid.h" #include "eckit/geo/util/mutex.h" #include "eckit/log/Log.h" namespace eckit::geo::search { Tree::Point::value_type Tree::Point::distance(const Point& p, const Point& q) { value_type d2 = 0; for (size_t i = 0; i < DIMS; ++i) { d2 += (p[i] - q[i]) * (p[i] - q[i]); } return std::sqrt(d2); } Tree::Point::value_type Tree::Point::distance(const Point& p, const Point& q, size_t axis) { return std::abs(p[axis] - q[axis]); } Tree::~Tree() = default; Tree::Tree(const Grid& r) : itemCount_(r.size()) { ASSERT(itemCount_ > 0); } void Tree::build(std::vector& /*unused*/) { throw exception::SeriousBug("Tree::build() not implemented for " + str()); } void Tree::insert(const PointValueType& /*unused*/) { throw exception::SeriousBug("Tree::insert() not implemented for " + str()); } void Tree::statsPrint(std::ostream& /*unused*/, bool /*unused*/) { throw exception::SeriousBug("Tree::statsPrint() not implemented for " + str()); } void Tree::statsReset() { throw exception::SeriousBug("Tree::statsReset() not implemented for " + str()); } Tree::PointValueType Tree::nearestNeighbour(const Point& /*unused*/) { throw exception::SeriousBug("Tree::nearestNeighbour() not implemented for " + str()); } std::vector Tree::kNearestNeighbours(const Point& /*unused*/, size_t /*unused*/) { throw exception::SeriousBug("Tree::kNearestNeighbours() not implemented for " + str()); } std::vector Tree::findInSphere(const Point& /*unused*/, double /*unused*/) { throw exception::SeriousBug("Tree::findInSphere() not implemented for " + str()); } bool Tree::ready() const { throw exception::SeriousBug("Tree::ready() not implemented for " + str()); } void Tree::commit() { throw exception::SeriousBug("Tree::commit() not implemented for " + str()); } void Tree::print(std::ostream& out) const { out << "Tree[]" << std::endl; } void Tree::lock() { // Empty } void Tree::unlock() { // Empty } std::string Tree::str() const { std::ostringstream os; print(os); return os.str(); } static util::recursive_mutex MUTEX; static std::map TREES; TreeFactory::TreeFactory(const std::string& name) : name_(name) { util::lock_guard lock(MUTEX); if (auto j = TREES.find(name); j == TREES.end()) { TREES[name] = this; return; } throw exception::SeriousBug("TreeFactory: duplicate '" + name + "'"); } TreeFactory::~TreeFactory() { util::lock_guard lock(MUTEX); TREES.erase(name_); } Tree* TreeFactory::build(const std::string& name, const Grid& r) { util::lock_guard lock(MUTEX); if (auto j = TREES.find(name); j != TREES.end()) { return j->second->make(r); } list(Log::error() << "TreeFactory: unknown '" << name << "', choices are: "); throw exception::SeriousBug("TreeFactory: unknown '" + name + "'"); } void TreeFactory::list(std::ostream& out) { util::lock_guard lock(MUTEX); const auto* sep = ""; for (const auto& j : TREES) { out << sep << j.first; sep = ", "; } } } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/search/TreeMemory.h0000664000175000017500000000226315161702250021153 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/container/KDTree.h" #include "eckit/geo/search/Tree.h" namespace eckit::geo::search { class TreeMemory : public Tree { protected: KDTreeMemory tree_; void build(std::vector&) override; void insert(const PointValueType&) override; void statsPrint(std::ostream&, bool pretty) override; void statsReset() override; PointValueType nearestNeighbour(const Tree::Point&) override; std::vector kNearestNeighbours(const Point&, size_t k) override; std::vector findInSphere(const Point&, double radius) override; bool ready() const override; void commit() override; void print(std::ostream&) const override; public: using Tree::Tree; }; } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/search/TreeMappedFile.cc0000664000175000017500000000610415161702250022045 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/search/TreeMappedFile.h" #include "eckit/filesystem/PathExpander.h" #include "eckit/utils/Tokenizer.h" #include "eckit/geo/Exceptions.h" #include "eckit/geo/Grid.h" #include "eckit/geo/LibEcKitGeo.h" #include "eckit/log/Log.h" namespace eckit::geo::search { template PathName TreeMappedFile::treePath(const Grid& r, bool makeUnique) { // LocalPathName::unique and LocalPathName::mkdir call mkdir, make sure to use umask = 0 AutoUmask umask(0); static const long VERSION = 2; const std::string relative = "eckit/geo/search/" + std::to_string(VERSION) + "/" + r.uid() + ".kdtree"; auto writable = [](const PathName& path) -> bool { return (::access(path.asString().c_str(), W_OK) == 0); }; for (PathName path : T::roots()) { if (not path.exists()) { if (not writable(path.dirName())) { continue; } try { path.mkdir(0777); } catch (FailedSystemCall&) { // ignore } } if (not writable(path)) { Log::debug() << "TreeMappedFile: path '" << path << "' isn't writable" << std::endl; continue; } path /= relative; if (makeUnique && !path.exists()) { path = PathName::unique(path); } Log::debug() << "TreeMappedFile: path '" << path << "'" << (makeUnique ? " (unique)" : "") << std::endl; return path; } throw exception::SeriousBug("TreeMappedFile: no paths are viable for caching"); } template PathName TreeMappedFile::lockFile(const std::string& path) { AutoUmask umask(0); PathName lock(path + ".lock"); lock.touch(); return lock; } class TreeMappedCacheFile : public TreeMappedFile { using P = TreeMappedFile; public: using P::P; static std::vector roots() { static auto roots = []() { std::vector r; Tokenizer{":"}(LibEcKitGeo::cacheDir(), r); for (auto& root : r) { root = PathExpander::expand(root); } return r; }(); return roots; } }; static const TreeBuilder builder1("mapped-cache-file"); class TreeMappedTempFile : public TreeMappedFile { using P = TreeMappedFile; public: using P::P; static std::vector roots() { static std::vector _root{"/tmp"}; return _root; } }; static const TreeBuilder builder2("mapped-temporary-file"); } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/search/TreeMappedFile.h0000664000175000017500000000353715161702250021716 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/geo/search/TreeMapped.h" #include "eckit/os/Semaphore.h" #include "eckit/runtime/Main.h" namespace eckit::geo::search { template class TreeMappedFile : public TreeMapped { protected: PathName real_; Semaphore lock_; // Must be after real bool ready() const override { return path() == real_; } void commit() override { PathName::rename(path(), real_); } void print(std::ostream& out) const override { out << "TreeMappedFile[" "path=" << path() << ",ready?" << ready() << "]"; } void lock() override { AutoUmask umask(0); auto path = lockFile(real_); lock_.lock(); std::ofstream os(path.asString().c_str()); os << Main::hostname() << " " << ::getpid() << std::endl; } void unlock() override { PathName path = lockFile(real_); std::ofstream os(path.asString().c_str()); os << std::endl; lock_.unlock(); } static PathName treePath(const Grid& r, bool makeUnique); static PathName lockFile(const std::string& path); public: explicit TreeMappedFile(const Grid& r) : TreeMapped(r, treePath(r, true)), real_(treePath(r, false)), lock_(lockFile(real_)) { lockFile(real_).touch(); if (ready()) { Log::debug() << "Loading " << *this << std::endl; } } }; } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/search/TreeMemory.cc0000664000175000017500000000345215161702250021312 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/search/TreeMemory.h" namespace eckit::geo::search { void TreeMemory::build(std::vector& v) { tree_.build(v); } void TreeMemory::insert(const Tree::PointValueType& pt) { tree_.insert(pt); } void TreeMemory::statsPrint(std::ostream& out, bool pretty) { tree_.statsPrint(out, pretty); } void TreeMemory::statsReset() { tree_.statsReset(); } Tree::PointValueType TreeMemory::nearestNeighbour(const Tree::Point& pt) { const auto& nn = tree_.nearestNeighbour(pt).value(); return {nn.point(), nn.payload()}; } std::vector TreeMemory::kNearestNeighbours(const Tree::Point& pt, size_t k) { std::vector result; for (const auto& n : tree_.kNearestNeighbours(pt, k)) { result.emplace_back(PointValueType(n.point(), n.payload())); } return result; } std::vector TreeMemory::findInSphere(const Tree::Point& pt, double radius) { std::vector result; for (const auto& n : tree_.findInSphere(pt, radius)) { result.emplace_back(PointValueType(n.point(), n.payload())); } return result; } bool TreeMemory::ready() const { return false; } void TreeMemory::commit() {} void TreeMemory::print(std::ostream& out) const { out << "TreeMemory[]"; } static const TreeBuilder builder("memory"); } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/search/TreeMappedAnonymousMemory.cc0000664000175000017500000000131315161702250024344 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/search/TreeMappedAnonymousMemory.h" namespace eckit::geo::search { TreeMappedAnonymousMemory::TreeMappedAnonymousMemory(const Grid& r) : TreeMapped(r, "/dev/zero") {} static const TreeBuilder builder("mapped-anonymous-memory"); } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/search/Tree.h0000664000175000017500000000642515161702250017766 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/container/sptree/SPValue.h" #include "eckit/geo/PointXYZ.h" namespace eckit::geo { class Grid; } namespace eckit::geo::search { class Tree { public: #if 0 using Point = PointXYZ; #else struct Point : private std::array { static constexpr size_t DIMS = 3; Point(value_type x, value_type y, value_type z) : array{x, y, z} {} using array::array; explicit Point(const PointXYZ& p) : Point{p.X(), p.Y(), p.Z()} {} value_type x(size_t axis) const { return operator[](axis); } PointXYZ to_xyz() const { return {operator[](0), operator[](1), operator[](2)}; } // (additional) static value_type distance(const Point& p, const Point& q); static value_type distance(const Point& p, const Point& q, size_t axis); static constexpr value_type EPS = PointXYZ::EPS; friend std::ostream& operator<<(std::ostream& out, const Point& p) { return out << '{' << p[0] << ", " << p[1] << ", " << p[2] << '}'; } }; #endif using Payload = size_t; using PointValueType = SPValue; explicit Tree(const Grid&); Tree(const Tree&) = delete; Tree(Tree&&) = delete; virtual ~Tree(); Tree& operator=(const Tree&) = delete; Tree& operator=(Tree&&) = delete; virtual void build(std::vector&); virtual void insert(const PointValueType&); virtual void statsPrint(std::ostream&, bool); virtual void statsReset(); virtual PointValueType nearestNeighbour(const Point&); virtual std::vector kNearestNeighbours(const Point&, size_t k); virtual std::vector findInSphere(const Point&, double); virtual bool ready() const; virtual void commit(); virtual void print(std::ostream&) const; virtual void lock(); virtual void unlock(); std::string str() const; size_t itemCount() const { return itemCount_; } friend std::ostream& operator<<(std::ostream& s, const Tree& p) { p.print(s); return s; } private: const size_t itemCount_; }; class TreeFactory { protected: virtual Tree* make(const Grid&) = 0; explicit TreeFactory(const std::string&); virtual ~TreeFactory(); public: TreeFactory(const TreeFactory&) = delete; TreeFactory(TreeFactory&&) = delete; TreeFactory& operator=(TreeFactory&&) = delete; TreeFactory& operator=(const TreeFactory&) = delete; static Tree* build(const std::string&, const Grid&); static void list(std::ostream&); private: std::string name_; }; template class TreeBuilder : public TreeFactory { Tree* make(const Grid& r) override { return new T(r); } public: explicit TreeBuilder(const std::string& name) : TreeFactory(name) {} }; } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/search/TreeMapped.h0000664000175000017500000000261115161702250021106 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/container/KDTree.h" #include "eckit/geo/search/Tree.h" #include "eckit/os/AutoUmask.h" namespace eckit::geo::search { class TreeMapped : public Tree { private: AutoUmask umask_; // Must be first PathName path_; KDTreeMapped tree_; protected: PathName path() const { return path_; } void build(std::vector&) override; void insert(const PointValueType&) override; void statsPrint(std::ostream&, bool pretty) override; void statsReset() override; PointValueType nearestNeighbour(const Tree::Point&) override; std::vector kNearestNeighbours(const Point&, size_t k) override; std::vector findInSphere(const Point&, double radius) override; bool ready() const override = 0; void commit() override = 0; void print(std::ostream&) const override = 0; public: TreeMapped(const Grid&, const PathName&); }; } // namespace eckit::geo::search eckit-2.0.7/src/eckit/geo/PointXYZ.cc0000664000175000017500000000227415161702250017462 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/PointXYZ.h" #include #include "eckit/types/FloatCompare.h" namespace eckit::geo { PointXYZ::value_type PointXYZ::distance(const PointXYZ& p, size_t axis) const { return std::abs(x(axis) - p.x(axis)); } PointXYZ::value_type PointXYZ::distance(const PointXYZ& p) const { return std::sqrt(distance2(p)); } PointXYZ::value_type PointXYZ::distance2(const PointXYZ& p) const { return (X() - p.X()) * (X() - p.X()) + (Y() - p.Y()) * (Y() - p.Y()) + (Z() - p.Z()) * (Z() - p.Z()); } bool points_equal(const PointXYZ& a, const PointXYZ& b, PointXYZ::value_type eps) { return types::is_approximately_equal(a.X(), b.X(), eps) && types::is_approximately_equal(a.Y(), b.Y(), eps) && types::is_approximately_equal(a.Z(), b.Z(), eps); } } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/PointLonLatR.h0000664000175000017500000000620015161702250020136 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include namespace eckit::geo { class PointLonLat; } namespace eckit::geo { /** * @brief The PointLonLatR class * @details A point on a geographic coordinate system, in (longitude, latitude) coordinates [radian]. They are fully * circular in space (also latitude), longitude is typically limited to [0, 2 pi[ and latitude to [-pi, pi], with * normalisation functions available, as well as conversion to and from degree-based coordinates. */ class PointLonLatR final : protected std::array { public: // -- Types using container_type = array; using container_type::value_type; // -- Constructors PointLonLatR() : PointLonLatR(0., 0.) {} PointLonLatR(value_type lon, value_type lat) : container_type{lon, lat} {} PointLonLatR(const PointLonLatR& other) : container_type(other) {} PointLonLatR(PointLonLatR&& other) : container_type(other) {} // -- Destructor ~PointLonLatR() = default; // -- Operators PointLonLatR& operator=(const PointLonLatR& other) { container_type::operator=(other); return *this; } PointLonLatR& operator=(PointLonLatR&& other) { container_type::operator=(other); return *this; } // -- Members value_type lonr() const { return container_type::operator[](0); } value_type latr() const { return container_type::operator[](1); } // -- Methods using container_type::data; bool pole(value_type eps = EPS) const; static size_t dimensions() { return DIMS; } static value_type normalise_angle_to_minimum(value_type, value_type minimum); static value_type normalise_angle_to_maximum(value_type, value_type maximum); [[nodiscard]] static PointLonLatR make(value_type lonr, value_type latr, value_type lonr_minimum = 0., value_type eps = EPS); [[nodiscard]] static PointLonLatR make_from_lonlat(value_type lon, value_type lat, value_type lonr_minimum = 0.); PointLonLatR antipode() const { return make(lonr(), latr() + FLAT_ANGLE); } // -- Class members static constexpr size_t DIMS = 2; static constexpr value_type EPS = 1e-10; static constexpr value_type FULL_ANGLE = 2. * M_PI; static constexpr value_type FLAT_ANGLE = M_PI; static constexpr value_type RIGHT_ANGLE = M_PI_2; // -- Friends friend std::ostream& operator<<(std::ostream& out, const PointLonLatR& p) { return out << '{' << p.lonr() << ", " << p.latr() << '}'; } }; bool points_equal(const PointLonLatR&, const PointLonLatR&, PointLonLatR::value_type eps = PointLonLatR::EPS); extern const PointLonLatR NORTH_POLE_R; extern const PointLonLatR SOUTH_POLE_R; } // namespace eckit::geo eckit-2.0.7/src/eckit/geo/PointXY.cc0000664000175000017500000000265115161702250017327 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/geo/PointXY.h" #include #include "eckit/types/FloatCompare.h" namespace eckit::geo { static const PointXY ZERO; PointXY::value_type PointXY::norm() const { return distance(ZERO); } PointXY PointXY::normalize() const { const auto l = norm(); return types::is_approximately_equal(l, 0., EPS) ? ZERO : PointXY{X() / l, Y() / l}; } PointXY PointXY::middle(const PointXY& p) const { return (*this + p) * 0.5; } PointXY::value_type PointXY::distance(const PointXY& p, size_t axis) const { return std::abs(x(axis) - p.x(axis)); } PointXY::value_type PointXY::distance(const PointXY& p) const { return std::sqrt(distance2(p)); } PointXY::value_type PointXY::distance2(const PointXY& p) const { return (X() - p.X()) * (X() - p.X()) + (Y() - p.Y()) * (Y() - p.Y()); } bool points_equal(const PointXY& a, const PointXY& b, PointXY::value_type eps) { return types::is_approximately_equal(a.X(), b.X(), eps) && types::is_approximately_equal(a.Y(), b.Y(), eps); } } // namespace eckit::geo eckit-2.0.7/src/eckit/memory/0000775000175000017500000000000015161702250016200 5ustar alastairalastaireckit-2.0.7/src/eckit/memory/Builder.h0000664000175000017500000002143315161702250017742 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Builder.h /// @author Tiago Quintino /// @author Pedro Maciel /// @date Jul 2014 #pragma once #include "eckit/eckit_config.h" #include "eckit/memory/Factory.h" #if eckit_HAVE_ECKIT_MEMORY_FACTORY_BUILDERS_DEBUG #include "eckit/exception/Exceptions.h" #define DEBUG_BUILDER(x) std::cerr << " DEBUG (" << x << ") " << Here() << std::endl; #else #define DEBUG_BUILDER(x) #endif namespace eckit { //------------------------------------------------------------------------------------------------------ class Builder { public: // -- Types using key_t = std::string; // -- Constructors Builder() = default; Builder(const Builder&) = delete; Builder(Builder&&) = delete; // -- Destructor virtual ~Builder() = default; // -- Operators void operator=(const Builder&) = delete; void operator=(Builder&&) = delete; // -- Methods virtual key_t name() const = 0; virtual key_t build_type() const = 0; private: // -- Methods virtual void print(std::ostream& os) const { os << "Builder(" << build_type() << "):" << name(); } // -- Friends friend std::ostream& operator<<(std::ostream& os, const Builder& o) { o.print(os); return os; } }; //------------------------------------------------------------------------------------------------------ template class BuilderT0 : public Builder { public: // -- Types using product_t = Base; // -- Methods virtual product_t* create() const = 0; // -- Overridden methods typename Builder::key_t build_type() const override { return Base::className(); } }; //------------------------------------------------------------------------------------------------------ template class BuilderT1 : public Builder { public: // -- Types using product_t = Base; using ARG1 = typename product_t::ARG1; // -- Methods virtual product_t* create(ARG1) const = 0; // -- Overridden methods typename Builder::key_t build_type() const override { return Base::className(); } }; //------------------------------------------------------------------------------------------------------ template class BuilderT2 : public Builder { public: // -- Types using product_t = Base; using ARG1 = typename product_t::ARG1; using ARG2 = typename product_t::ARG2; // -- Methods virtual product_t* create(ARG1, ARG2) const = 0; // -- Overridden methods typename Builder::key_t build_type() const override { return Base::className(); } }; //------------------------------------------------------------------------------------------------------ template class ConcreteBuilderT0 final : public BuilderT0 { public: // -- Types using base_t = BuilderT0; // -- Constructors ConcreteBuilderT0() : key_(name()) { DEBUG_BUILDER("ConcreteBuilderT0() -- " << T::className()); Factory::instance().regist(key_, this); } explicit ConcreteBuilderT0(const typename base_t::key_t& k) : key_(k) { DEBUG_BUILDER("ConcreteBuilderT0() -- " << T::className()); Factory::instance().regist(key_, this); } ConcreteBuilderT0(const ConcreteBuilderT0&) = delete; ConcreteBuilderT0(ConcreteBuilderT0&&) = delete; // -- Destructor ~ConcreteBuilderT0() override { DEBUG_BUILDER("~ConcreteBuilderT0() -- " << T::className()); #if eckit_HAVE_ECKIT_MEMORY_FACTORY_EMPTY_DESTRUCTION Factory::instance().unregist(key_); #endif } // -- Operators void operator=(const ConcreteBuilderT0&) = delete; void operator=(ConcreteBuilderT0&&) = delete; // -- Overridden methods typename base_t::key_t name() const override { return T::className(); } typename base_t::product_t* create() const override { return new T(); } private: // -- Members typename base_t::key_t key_; }; #define register_BuilderT0(ABSTRACT, CONCRETE, NAME) \ static struct Register__##ABSTRACT##__##CONCRETE##__T0 { \ Register__##ABSTRACT##__##CONCRETE##__T0() { \ static eckit::ConcreteBuilderT0 builder(NAME); \ } \ } register_##ABSTRACT##__##CONCRETE##_T0 //------------------------------------------------------------------------------------------------------ template class ConcreteBuilderT1 final : public BuilderT1 { public: // -- Types using base_t = BuilderT1; // -- Constructors ConcreteBuilderT1() : key_(name()) { DEBUG_BUILDER("ConcreteBuilderT1() -- " << T::className()); Factory::instance().regist(key_, this); } explicit ConcreteBuilderT1(const typename base_t::key_t& k) : key_(k) { DEBUG_BUILDER("ConcreteBuilderT1() -- " << T::className()); Factory::instance().regist(key_, this); } ConcreteBuilderT1(const ConcreteBuilderT1&) = delete; ConcreteBuilderT1(ConcreteBuilderT1&&) = delete; // -- Destructor ~ConcreteBuilderT1() override { DEBUG_BUILDER("~ConcreteBuilderT1() -- " << T::className()); #if ECKIT_MEMORY_FACTORY_EMPTY_DESTRUCTION Factory::instance().unregist(key_); #endif } // -- Operators void operator=(const ConcreteBuilderT1&) = delete; void operator=(ConcreteBuilderT1&&) = delete; // -- Overridden methods typename base_t::key_t name() const override { return T::className(); } typename base_t::product_t* create(typename base_t::ARG1 p1) const override { return new T(p1); } private: // -- Members typename base_t::key_t key_; }; #define register_BuilderT1(ABSTRACT, CONCRETE, NAME) \ static struct Register__##ABSTRACT##__##CONCRETE##__T1 { \ Register__##ABSTRACT##__##CONCRETE##__T1() { \ static eckit::ConcreteBuilderT1 builder(NAME); \ } \ } register_##ABSTRACT##__##CONCRETE##_T1 //------------------------------------------------------------------------------------------------------ template class ConcreteBuilderT2 final : public BuilderT2 { public: // -- Types using base_t = BuilderT2; // -- Constructors ConcreteBuilderT2() : key_(name()) { DEBUG_BUILDER("ConcreteBuilderT2() -- " << T::className()); Factory::instance().regist(key_, this); } explicit ConcreteBuilderT2(const typename base_t::key_t& k) : key_(k) { DEBUG_BUILDER("ConcreteBuilderT2() -- " << T::className()); Factory::instance().regist(key_, this); } ConcreteBuilderT2(const ConcreteBuilderT2&) = delete; ConcreteBuilderT2(ConcreteBuilderT2&&) = delete; // -- Destructor ~ConcreteBuilderT2() override { DEBUG_BUILDER("~ConcreteBuilderT2() -- " << T::className()); #if eckit_HAVE_ECKIT_MEMORY_FACTORY_EMPTY_DESTRUCTION Factory::instance().unregist(key_); #endif } // -- Operators void operator=(const ConcreteBuilderT2&) = delete; void operator=(ConcreteBuilderT2&&) = delete; // -- Overridden methods typename base_t::key_t name() const override { return T::className(); } typename base_t::product_t* create(typename base_t::ARG1 p1, typename base_t::ARG2 p2) const override { return new T(p1, p2); } private: // -- Members typename base_t::key_t key_; }; #define register_BuilderT2(ABSTRACT, CONCRETE, NAME) \ static struct Register__##ABSTRACT##__##CONCRETE##__T2 { \ Register__##ABSTRACT##__##CONCRETE##__T2() { \ static eckit::ConcreteBuilderT2 builder(NAME); \ } \ } register_##ABSTRACT##__##CONCRETE##_T2 //------------------------------------------------------------------------------------------------------ } // namespace eckit eckit-2.0.7/src/eckit/memory/NonCopyable.h0000664000175000017500000000141115161702250020557 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_NonCopyable_h #define eckit_NonCopyable_h namespace eckit { /// Inherit from this class to make a NonCopyable class class NonCopyable { protected: NonCopyable(); ~NonCopyable(); private: // No copy allowed NonCopyable(const NonCopyable&) = delete; NonCopyable& operator=(const NonCopyable&) = delete; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/memory/MemoryBuffer.cc0000664000175000017500000000333415161702250021114 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/memory/MemoryBuffer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- MemoryBuffer::MemoryBuffer(size_t size) : buffer_{nullptr}, size_(size) { create(); } MemoryBuffer::MemoryBuffer(const char* p, size_t size) : buffer_{nullptr}, size_(size) { create(); copy(p, size); } MemoryBuffer::MemoryBuffer(const std::string& s) : buffer_{nullptr}, size_(s.length() + 1) { create(); copy(s); } MemoryBuffer::~MemoryBuffer() { destroy(); } void MemoryBuffer::create() { buffer_ = new char[size_]; ASSERT(buffer_); } void MemoryBuffer::destroy() { char* p = static_cast(buffer_); delete[] p; } void MemoryBuffer::copy(const std::string& s) { ::strcpy(static_cast(buffer_), s.c_str()); } void MemoryBuffer::copy(const char* p, size_t size) { ::memcpy(buffer_, p, size); } void MemoryBuffer::swap(MemoryBuffer& rhs) { std::swap(buffer_, rhs.buffer_); std::swap(size_, rhs.size_); } void eckit::MemoryBuffer::resize(size_t size) { destroy(); size_ = size; create(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/memory/Counted.h0000664000175000017500000000436215161702250017757 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Counted.h /// @date Jun 1996 /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_Counted_h #define eckit_Counted_h #include "eckit/thread/Mutex.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- namespace memory::detail { class ThreadedLock { public: void lock() const { mutex_.lock(); } void unlock() const { mutex_.unlock(); } mutable Mutex mutex_; }; class NoLock { public: void lock() const {} void unlock() const {} }; } // namespace memory::detail //---------------------------------------------------------------------------------------------------------------------- /// Reference counting objects /// Subclass from this class if you want reference counting object. /// @note Remember to use 'virtual' inheritance in case of multiple inheritance class Counted : private memory::detail::ThreadedLock { public: // methods void attach() const { lock(); count_++; unlock(); } void detach() const { lock(); if (--count_ == 0) { unlock(); delete this; } else { unlock(); } } size_t count() const { return count_; } void lock() const { memory::detail::ThreadedLock::lock(); } void unlock() const { memory::detail::ThreadedLock::unlock(); } public: Counted() : count_(0) {} Counted(const Counted&) = delete; Counted& operator=(const Counted&) = delete; Counted(Counted&&) = delete; Counted& operator=(Counted&&) = delete; virtual ~Counted(); private: // members mutable size_t count_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/memory/MMap.cc0000664000175000017500000000364715161702250017353 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/BigNum.h" #include "eckit/log/Bytes.h" #include "eckit/memory/MMap.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static eckit::StaticMutex local_mutex; static long count_; static long maxCount_; static size_t length_; static size_t maxLength_; //---------------------------------------------------------------------------------------------------------------------- void* MMap::mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) { void* r = ::mmap(addr, length, prot, flags, fd, offset); if (r != MAP_FAILED) { AutoLock lock(local_mutex); count_++; maxCount_ = std::max(count_, maxCount_); length_ += length; maxLength_ = std::max(length_, maxLength_); } return r; } int MMap::munmap(void* addr, size_t length) { int r = ::munmap(addr, length); if (r == 0) { AutoLock lock(local_mutex); count_--; length_ -= length; } return r; } void MMap::info(size_t& count, size_t& size) { AutoLock lock(local_mutex); count = count_; size = length_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/memory/Factory.h0000664000175000017500000001152615161702250017765 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Factory.h /// @author Tiago Quintino /// @author Pedro Maciel /// @date Jul 2014 #pragma once #include #include #include #include "eckit/eckit_config.h" #include "eckit/exception/Exceptions.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" namespace eckit { //------------------------------------------------------------------------------------------------------ template class Factory { public: // -- Types using product_t = T; using builder_t = typename product_t::builder_t; using key_t = std::string; using storage_t = std::map; // -- Constructors Factory(const Factory&) = delete; Factory(Factory&&) = delete; // -- Operators void operator=(const Factory&) = delete; void operator=(Factory&&) = delete; // -- Methods /// @return the instance of this singleton factory static Factory& instance(); /// @returns class name of the type built by this factory static std::string build_type() { return T::className(); } /// Checks if a builder is registered /// @param name of the builder bool exists(const key_t&) const; /// Registers a builder /// @param builder pointer /// @throw BadParameter if the builder already registered void regist(const key_t&, builder_t*); /// Remove a registered builder /// @throw BadParameter if the builder is not registered void unregist(const key_t&); /// Gets the builder registered to the associated key /// @param name of the builder const builder_t& get(const key_t&) const; /// @returns the number of builders registered to the factory size_t size() const; /// @returns the builder keys registered to the factory std::vector keys() const; private: // -- Constructors Factory() = default; #if eckit_HAVE_ECKIT_MEMORY_FACTORY_EMPTY_DESTRUCTION // -- Destructor ~Factory() { ASSERT(store_.empty()); } #endif // -- Members mutable Mutex mutex_; ///< mutex protecting Factory singleton storage_t store_; ///< storage for the builders in a map indexed by key_t // -- Methods void print(std::ostream&) const; // -- Friends friend std::ostream& operator<<(std::ostream& os, const Factory& o) { o.print(os); return os; } }; //------------------------------------------------------------------------------------------------------ template Factory& Factory::instance() { static Factory obj; return obj; } template bool Factory::exists(const key_t& k) const { AutoLock lock(mutex_); return (store_.find(k) != store_.end()); } template void Factory::regist(const key_t& k, builder_t* b) { AutoLock lock(mutex_); ASSERT(b != nullptr); if (exists(k)) { throw BadParameter("Factory(" + build_type() + ") has already a builder for " + k, Here()); } store_[k] = b; } template void Factory::unregist(const key_t& k) { AutoLock lock(mutex_); if (!exists(k)) { throw BadParameter("Factory(" + build_type() + ") has no builder for " + k, Here()); } store_.erase(k); } template size_t Factory::size() const { AutoLock lock(mutex_); return store_.size(); } template const typename Factory::builder_t& Factory::get(const key_t& k) const { AutoLock lock(mutex_); if (!exists(k)) { throw BadParameter("Factory(" + build_type() + ") has no builder for " + k, Here()); } return *(store_.find(k)->second); } template void Factory::print(std::ostream& os) const { AutoLock lock(mutex_); os << "Factory(" << build_type() << ")" << std::endl; int key_width = 0; for (const auto& i : store_) { key_width = std::max(static_cast(i.first.size()), key_width); } for (const auto& i : store_) { os << " " << std::setw(key_width) << std::left << i.first << " -- " << i.second << std::endl; } } template std::vector::key_t> Factory::keys() const { AutoLock lock(mutex_); std::vector keysv; keysv.reserve(store_.size()); for (const auto& i : store_) { keysv.push_back(i.first); } return keysv; } //------------------------------------------------------------------------------------------------------ } // namespace eckit eckit-2.0.7/src/eckit/memory/Counted.cc0000664000175000017500000000134115161702250020107 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/memory/Counted.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Counted::~Counted() {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/memory/MapAllocator.cc0000664000175000017500000000572515161702250021076 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/memory/MapAllocator.h" #include "eckit/memory/MMap.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static size_t whole_page(size_t size) { static size_t page = sysconf(_SC_PAGE_SIZE); return ((size + page - 1) / page) * page; } MapAllocatorTooSmall::MapAllocatorTooSmall(size_t, size_t) : Exception("MapAllocator too small") {} MapAllocator::MapAllocator(size_t length) : fd_(-1), length_(whole_page(length)), count_(0), more_{nullptr} { addr_ = MMap::mmap(nullptr, length_, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, fd_, 0); if (addr_ == MAP_FAILED) { throw FailedSystemCall("mmap", Here()); } next_ = (char*)addr_; left_ = length_; // Log::warning() << "MapAllocator created " << length_ << std::endl; } MapAllocator::~MapAllocator() { // Log::warning() << "MapAllocator deleted " << length_ << std::endl; munmap(addr_, length_); if (fd_ >= 0) { close(fd_); } delete more_; } union Align { char char_; double double_; long long longlong_; void* voidstar_; float float_; }; const int WORD = sizeof(Align); void* MapAllocator::allocate(size_t size) { // return ::operator new(size); size = ((size + WORD - 1) / WORD) * WORD; if (size > left_) { if (!more_) { // Log::warning() << "MapAllocator too small: length=" << length_ << " left=" << left_ << " // request=" //<< size << std::endl; more_ = new MapAllocator(std::max(length_, size)); ASSERT(more_); } return more_->allocate(size); } char* addr = next_; next_ += size; left_ -= size; count_++; return addr; } void MapAllocator::deallocate(void* addr) { //::operator delete(addr); return; char* p = (char*)addr; char* q = (char*)addr_; if (p >= q && p < q + length_) { count_--; if (count_ == 0) { // Log::warning() << "MapAllocator empty" << std::endl; next_ = (char*)addr_; left_ = length_; } } else { ASSERT(more_); more_->deallocate(addr); if (more_->count_ == 0 && more_->more_ == nullptr) { delete more_; more_ = nullptr; } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/memory/Zero.h0000664000175000017500000000126115161702250017270 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ ///@author Baudouin Raoult ///@author Tiago Quintino ///@date Dec 2018 #ifndef eckit_memory_Zero_h #define eckit_memory_Zero_h #include // for memset namespace eckit { template inline void zero(T& p) { ::memset(&p, 0, sizeof(T)); } } // namespace eckit #endif eckit-2.0.7/src/eckit/memory/MapAllocator.h0000664000175000017500000000314715161702250020734 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File MapAllocator.h // MARS (Baudouin Raoult) - ECMWF Nov 01 #ifndef eckit_MapAllocator_h #define eckit_MapAllocator_h #include "eckit/exception/Exceptions.h" #include "eckit/types/Types.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class MapAllocatorTooSmall : public Exception { public: MapAllocatorTooSmall(size_t, size_t); }; //---------------------------------------------------------------------------------------------------------------------- class MapAllocator { public: MapAllocator(size_t); MapAllocator(const MapAllocator&) = delete; MapAllocator& operator=(const MapAllocator&) = delete; MapAllocator(MapAllocator&&) = delete; MapAllocator& operator=(MapAllocator&&) = delete; ~MapAllocator(); void* allocate(size_t); void deallocate(void*); private: // members int fd_; void* addr_; char* next_; size_t length_; size_t left_; Ordinal count_; MapAllocator* more_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/memory/Shmget.h0000664000175000017500000000216315161702250017602 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Dec 2017 #ifndef eckit_memory_Shmget_h #define eckit_memory_Shmget_h #include // for key_t #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Shmget { public: // methods static int shmget(key_t key, size_t size, int shmflg); static void* shmat(int shmid, const void* shmaddr, int shmflg); static int shmdt(const void* shmaddr); static void info(size_t& count, size_t& size); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/memory/Shmget.cc0000664000175000017500000000441515161702250017742 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/BigNum.h" #include "eckit/log/Bytes.h" #include "eckit/memory/Shmget.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static eckit::StaticMutex local_mutex; static long count_; static long maxCount_; static size_t length_; static size_t maxLength_; static std::map sizes_; static std::map ids_; //---------------------------------------------------------------------------------------------------------------------- int Shmget::shmget(key_t key, size_t size, int shmflg) { int shmid = ::shmget(key, size, shmflg); if (shmid >= 0) { AutoLock lock(local_mutex); sizes_[shmid] = size; } return shmid; } void* Shmget::shmat(int shmid, const void* shmaddr, int shmflg) { void* addr = ::shmat(shmid, shmaddr, shmflg); if (addr) { AutoLock lock(local_mutex); count_++; maxCount_ = std::max(count_, maxCount_); length_ += sizes_[shmid]; maxLength_ = std::max(length_, maxLength_); ids_[addr] = shmid; } return addr; } int Shmget::shmdt(const void* shmaddr) { int code = ::shmdt(shmaddr); if (code == 0) { AutoLock lock(local_mutex); count_--; length_ -= sizes_[ids_[shmaddr]]; sizes_.erase(ids_[shmaddr]); ids_.erase(shmaddr); } return code; } void Shmget::info(size_t& count, size_t& size) { AutoLock lock(local_mutex); count = count_; size = length_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/memory/MemoryBuffer.h0000664000175000017500000000367015161702250020761 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date July 1996 #ifndef eckit_memory_MemoryBlock_h #define eckit_memory_MemoryBlock_h #include #include "eckit/eckit.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- // A simple class to implement buffers class MemoryBuffer { public: // methods MemoryBuffer(size_t size); MemoryBuffer(const std::string& s); MemoryBuffer(const char*, size_t size); MemoryBuffer(const MemoryBuffer&) = delete; MemoryBuffer& operator=(const MemoryBuffer&) = delete; MemoryBuffer(MemoryBuffer&&) = delete; MemoryBuffer& operator=(MemoryBuffer&&) = delete; ~MemoryBuffer(); operator char*() { return static_cast(buffer_); } operator const char*() const { return static_cast(buffer_); } operator void*() { return buffer_; } operator const void*() const { return buffer_; } void* data() { return buffer_; } const void* data() const { return buffer_; } size_t size() const { return size_; } void resize(size_t size); void swap(MemoryBuffer& rhs); protected: // methods void create(); void destroy(); void copy(const std::string& s); void copy(const char*, size_t size); private: // members void* buffer_; size_t size_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/memory/Owned.h0000664000175000017500000000363515161702250017434 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Owned.h /// @author Tiago Quintino /// @date May 2014 #ifndef eckit_memory_Owned_h #define eckit_memory_Owned_h #include "eckit/memory/Counted.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// Reference counting objects /// Subclass from this class to use a SharedPtr class template class OwnedT : public LOCK { public: // methods OwnedT() : count_(0) {} OwnedT(const OwnedT&) = delete; OwnedT& operator=(const OwnedT&) = delete; OwnedT(OwnedT&&) = delete; OwnedT& operator=(OwnedT&&) = delete; virtual ~OwnedT() {} void attach() const { LOCK::lock(); count_++; LOCK::unlock(); } void detach() const { LOCK::lock(); --count_; LOCK::unlock(); } size_t owners() const { return count_; } private: // members mutable size_t count_; }; //---------------------------------------------------------------------------------------------------------------------- /// Owned object without thread lockable resource using OwnedLock = OwnedT; /// Owned object with thread lockable resource using OwnedNoLock = OwnedT; /// Default Owned type /// Same as OwnedNoLock using Owned = OwnedNoLock; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/memory/MMap.h0000664000175000017500000000205715161702250017207 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Dec 2017 #ifndef eckit_memory_MMap_h #define eckit_memory_MMap_h #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class MMap { public: // methods static void* mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset); static int munmap(void* addr, size_t length); static void info(size_t& count, size_t& size); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/memory/Padded.h0000664000175000017500000000230115161702250017526 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once // File Padded.h // Baudouin Raoult - ECMWF Jan 97 #include //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- /// Allows to pad T to the next smallest multiple of size larger or equal than sizeof(T). /// @warning This class is often written to disk! Any change must ensure binary compatibility. template class Padded : public T { private: static constexpr auto align_ = size; static constexpr auto osize_ = sizeof(T); // Add the padding char padding_[((size_t(osize_) + size_t(align_) - 1) / align_) * align_ - osize_]{}; }; //----------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/memory/NonCopyable.cc0000664000175000017500000000160615161702250020723 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/memory/NonCopyable.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- NonCopyable::NonCopyable() {} NonCopyable::~NonCopyable() {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/memory/OnlyMovable.h0000664000175000017500000000162015161702250020577 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_OnlyMovable_h #define eckit_OnlyMovable_h namespace eckit { /// Inherit from this class to make a OnlyMovable class class OnlyMovable { protected: OnlyMovable() {} ~OnlyMovable() {} // Copying not allowed OnlyMovable(const OnlyMovable&) = delete; OnlyMovable& operator=(const OnlyMovable&) = delete; // Moving is allowed OnlyMovable(OnlyMovable&&) = default; OnlyMovable& operator=(OnlyMovable&&) = default; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/utils/0000775000175000017500000000000015161702250016030 5ustar alastairalastaireckit-2.0.7/src/eckit/utils/StringTools.h0000664000175000017500000000536115161702250020475 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_StringTools_h #define eckit_StringTools_h #include #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class StringTools { public: StringTools() = delete; static std::string substitute(const std::string&, const std::map&); static std::vector listVariables(const std::string&); static std::string upper(const std::string&); static std::string lower(const std::string&); static std::string trim(const std::string&); static std::string trim(const std::string&, const std::string&); static std::string front_trim(const std::string&); static std::string front_trim(const std::string&, const std::string&); static std::string back_trim(const std::string&); static std::string back_trim(const std::string&, const std::string&); static std::vector split(const std::string& delim, const std::string& text); template static std::string join(const std::string&, const T&); template static std::string join(const std::string&, Iterator begin, Iterator end); static bool startsWith(const std::string& str, const std::string& substr); static bool beginsWith(const std::string& str, const std::string& substr); static bool endsWith(const std::string& str, const std::string& substr); static bool isQuoted(const std::string& value); static std::string unQuote(const std::string& value); }; //---------------------------------------------------------------------------------------------------------------------- template std::string StringTools::join(const std::string& delimiter, Iterator begin, Iterator end) { if (begin == end) { return ""; } std::string r(*begin); for (Iterator it = ++begin; it != end; ++it) { r += delimiter; r += *it; } return r; } template std::string StringTools::join(const std::string& delimiter, const T& words) { return join(delimiter, words.begin(), words.end()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/utils/Hash.h0000664000175000017500000001041015161702250017060 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_Hash_H #define eckit_utils_Hash_H #include "eckit/eckit.h" #include #include #include #include #include #include "eckit/thread/Mutex.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Hash { public: // types using digest_t = std::string; public: // methods Hash(); Hash(const Hash&) = delete; Hash& operator=(const Hash&) = delete; Hash(Hash&&) = delete; Hash& operator=(Hash&&) = delete; virtual ~Hash(); virtual void reset() const = 0; virtual digest_t digest() const = 0; // for one shot, stateless computation of the hash of the buffer virtual digest_t compute(const void*, long) = 0; void add(char x) { update(&x, sizeof(x)); } void add(unsigned char x) { update(&x, sizeof(x)); } void add(bool x) { update(&x, sizeof(x)); } void add(int x) { update(&x, sizeof(x)); } void add(unsigned int x) { update(&x, sizeof(x)); } void add(short x) { update(&x, sizeof(x)); } void add(unsigned short x) { update(&x, sizeof(x)); } void add(long x) { update(&x, sizeof(x)); } void add(unsigned long x) { update(&x, sizeof(x)); } void add(long long x) { update(&x, sizeof(x)); } void add(unsigned long long x) { update(&x, sizeof(x)); } void add(float x) { update(&x, sizeof(x)); } void add(double x) { update(&x, sizeof(x)); } void add(const void* x, long size) { update(x, size); } void add(const std::string& x) { update(x.c_str(), x.size()); } void add(const char* x) { update(x, std::strlen(x)); } template Hash& operator<<(const T& x) { add(x); return *this; } operator std::string() { return digest(); } protected: // methods // for incremental hashing virtual void update(const void*, long) = 0; private: // types // Make sure this is not called with a pointer template void add(const T* x); void add(const void*); /// Double hashing void add(const Hash& hash) { add(hash.digest()); } protected: // members mutable digest_t digest_; ///< cached digest }; //---------------------------------------------------------------------------------------------------------------------- class HashBuilderBase { std::string name_; public: HashBuilderBase(const std::string&); virtual ~HashBuilderBase(); virtual Hash* make() = 0; virtual Hash* make(const std::string& param) = 0; }; template class HashBuilder : public HashBuilderBase { Hash* make() override { return new T(); } Hash* make(const std::string& param) override { return new T(param); } public: HashBuilder(const std::string& name) : HashBuilderBase(name) {} ~HashBuilder() override = default; }; class HashFactory { public: static HashFactory& instance(); void add(const std::string& name, HashBuilderBase* builder); void remove(const std::string& name); bool has(const std::string& name); void list(std::ostream&); /// @returns default hash function Hash* build(); /** * @param name hash function name * @returns hash function built by specified builder */ Hash* build(const std::string& name); /** * @param name hash function name * @param param the initialization string, passed directly to the hash function constructor * @returns hash function built by specified builder */ Hash* build(const std::string& name, const std::string& param); private: HashFactory(); std::map builders_; eckit::Mutex mutex_; }; //---------------------------------------------------------------------------------------------------------------------- } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/MD5.cc0000664000175000017500000002756615161702250016744 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/exception/Exceptions.h" #include "eckit/utils/MD5.h" // Cray C++ compiler should not try to optimize this code #if _CRAYC #pragma _CRI noopt #endif namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /* Constants for MD5Transform routine. */ #define S11 7 #define S12 12 #define S13 17 #define S14 22 #define S21 5 #define S22 9 #define S23 14 #define S24 20 #define S31 4 #define S32 11 #define S33 16 #define S34 23 #define S41 6 #define S42 10 #define S43 15 #define S44 21 static unsigned char PADDING[64] = {0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; /* F, G, H and I are basic MD5 functions. */ #define F(x, y, z) (((x) & (y)) | ((~x) & (z))) #define G(x, y, z) (((x) & (z)) | ((y) & (~z))) #define H(x, y, z) ((x) ^ (y) ^ (z)) #define I(x, y, z) ((y) ^ ((x) | (~z))) /* ROTATE_LEFT rotates x left n bits. */ #define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) /* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4. Rotation is separate from addition to prevent recomputation. */ #define FF(a, b, c, d, x, s, ac) \ { \ (a) += F((b), (c), (d)) + (x) + (UINT4)(ac); \ (a) = ROTATE_LEFT((a), (s)); \ (a) += (b); \ } #define GG(a, b, c, d, x, s, ac) \ { \ (a) += G((b), (c), (d)) + (x) + (UINT4)(ac); \ (a) = ROTATE_LEFT((a), (s)); \ (a) += (b); \ } #define HH(a, b, c, d, x, s, ac) \ { \ (a) += H((b), (c), (d)) + (x) + (UINT4)(ac); \ (a) = ROTATE_LEFT((a), (s)); \ (a) += (b); \ } #define II(a, b, c, d, x, s, ac) \ { \ (a) += I((b), (c), (d)) + (x) + (UINT4)(ac); \ (a) = ROTATE_LEFT((a), (s)); \ (a) += (b); \ } /* MD5 initialization. Begins an MD5 operation, writing a new context. */ void MD5::Init(MD5_CTX* context) { context->count[0] = context->count[1] = 0; /* Load magic initialization constants.*/ context->state[0] = 0x67452301; context->state[1] = 0xefcdab89; context->state[2] = 0x98badcfe; context->state[3] = 0x10325476; } /* MD5 block update operation. Continues an MD5 message-digest operation, processing another message block, and updating the context. */ void MD5::Update(MD5_CTX* context, const unsigned char* input, unsigned int inputLen) { unsigned int i, index, partLen; /* Compute number of bytes mod 64 */ index = (unsigned int)((context->count[0] >> 3) & 0x3F); /* Update number of bits */ if ((context->count[0] += ((UINT4)inputLen << 3)) < ((UINT4)inputLen << 3)) { context->count[1]++; } context->count[1] += ((UINT4)inputLen >> 29); partLen = 64 - index; /* Transform as many times as possible. */ if (inputLen >= partLen) { ::memcpy((POINTER)&context->buffer[index], (POINTER)input, partLen); MD5::Transform(context->state, context->buffer); for (i = partLen; i + 63 < inputLen; i += 64) { MD5::Transform(context->state, &input[i]); } index = 0; } else { i = 0; } /* Buffer remaining input */ ::memcpy((POINTER)&context->buffer[index], (POINTER)&input[i], inputLen - i); } /* MD5 finalization. Ends an MD5 message-digest operation, writing the the message digest and zeroizing the context. */ void MD5::Final(unsigned char digest[MD5_DIGEST_LENGTH], MD5_CTX* context) { unsigned char bits[8]; unsigned int index, padLen; /* Save number of bits */ Encode(bits, context->count, 8); /* Pad out to 56 mod 64. */ index = (unsigned int)((context->count[0] >> 3) & 0x3f); padLen = (index < 56) ? (56 - index) : (120 - index); MD5::Update(context, PADDING, padLen); /* Append length (before padding) */ MD5::Update(context, bits, 8); /* Store state in digest */ Encode(digest, context->state, 16); /* Zeroize sensitive information.*/ ::memset((POINTER)context, 0, sizeof(*context)); } /* MD5 basic transformation. Transforms state based on block. */ void MD5::Transform(UINT4 state[4], const unsigned char block[64]) { UINT4 a = state[0]; UINT4 b = state[1]; UINT4 c = state[2]; UINT4 d = state[3]; UINT4 x[16]; ::memset((POINTER)x, 0, sizeof(x)); Decode(x, block, 64); /* Round 1 */ FF(a, b, c, d, x[0], S11, 0xd76aa478); /* 1 */ FF(d, a, b, c, x[1], S12, 0xe8c7b756); /* 2 */ FF(c, d, a, b, x[2], S13, 0x242070db); /* 3 */ FF(b, c, d, a, x[3], S14, 0xc1bdceee); /* 4 */ FF(a, b, c, d, x[4], S11, 0xf57c0faf); /* 5 */ FF(d, a, b, c, x[5], S12, 0x4787c62a); /* 6 */ FF(c, d, a, b, x[6], S13, 0xa8304613); /* 7 */ FF(b, c, d, a, x[7], S14, 0xfd469501); /* 8 */ FF(a, b, c, d, x[8], S11, 0x698098d8); /* 9 */ FF(d, a, b, c, x[9], S12, 0x8b44f7af); /* 10 */ FF(c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */ FF(b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */ FF(a, b, c, d, x[12], S11, 0x6b901122); /* 13 */ FF(d, a, b, c, x[13], S12, 0xfd987193); /* 14 */ FF(c, d, a, b, x[14], S13, 0xa679438e); /* 15 */ FF(b, c, d, a, x[15], S14, 0x49b40821); /* 16 */ /* Round 2 */ GG(a, b, c, d, x[1], S21, 0xf61e2562); /* 17 */ GG(d, a, b, c, x[6], S22, 0xc040b340); /* 18 */ GG(c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */ GG(b, c, d, a, x[0], S24, 0xe9b6c7aa); /* 20 */ GG(a, b, c, d, x[5], S21, 0xd62f105d); /* 21 */ GG(d, a, b, c, x[10], S22, 0x2441453); /* 22 */ GG(c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */ GG(b, c, d, a, x[4], S24, 0xe7d3fbc8); /* 24 */ GG(a, b, c, d, x[9], S21, 0x21e1cde6); /* 25 */ GG(d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */ GG(c, d, a, b, x[3], S23, 0xf4d50d87); /* 27 */ GG(b, c, d, a, x[8], S24, 0x455a14ed); /* 28 */ GG(a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ GG(d, a, b, c, x[2], S22, 0xfcefa3f8); /* 30 */ GG(c, d, a, b, x[7], S23, 0x676f02d9); /* 31 */ GG(b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ /* Round 3 */ HH(a, b, c, d, x[5], S31, 0xfffa3942); /* 33 */ HH(d, a, b, c, x[8], S32, 0x8771f681); /* 34 */ HH(c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */ HH(b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */ HH(a, b, c, d, x[1], S31, 0xa4beea44); /* 37 */ HH(d, a, b, c, x[4], S32, 0x4bdecfa9); /* 38 */ HH(c, d, a, b, x[7], S33, 0xf6bb4b60); /* 39 */ HH(b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */ HH(a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */ HH(d, a, b, c, x[0], S32, 0xeaa127fa); /* 42 */ HH(c, d, a, b, x[3], S33, 0xd4ef3085); /* 43 */ HH(b, c, d, a, x[6], S34, 0x4881d05); /* 44 */ HH(a, b, c, d, x[9], S31, 0xd9d4d039); /* 45 */ HH(d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */ HH(c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */ HH(b, c, d, a, x[2], S34, 0xc4ac5665); /* 48 */ /* Round 4 */ II(a, b, c, d, x[0], S41, 0xf4292244); /* 49 */ II(d, a, b, c, x[7], S42, 0x432aff97); /* 50 */ II(c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */ II(b, c, d, a, x[5], S44, 0xfc93a039); /* 52 */ II(a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */ II(d, a, b, c, x[3], S42, 0x8f0ccc92); /* 54 */ II(c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */ II(b, c, d, a, x[1], S44, 0x85845dd1); /* 56 */ II(a, b, c, d, x[8], S41, 0x6fa87e4f); /* 57 */ II(d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */ II(c, d, a, b, x[6], S43, 0xa3014314); /* 59 */ II(b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */ II(a, b, c, d, x[4], S41, 0xf7537e82); /* 61 */ II(d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */ II(c, d, a, b, x[2], S43, 0x2ad7d2bb); /* 63 */ II(b, c, d, a, x[9], S44, 0xeb86d391); /* 64 */ state[0] += a; state[1] += b; state[2] += c; state[3] += d; /* Zeroize sensitive information.*/ ::memset((POINTER)x, 0, sizeof(x)); } /* Encodes input (UINT4) into output (unsigned char). Assumes len is a multiple of 4. */ void MD5::Encode(unsigned char* output, UINT4* input, unsigned int len) { unsigned int i, j; for (i = 0, j = 0; j < len; i++, j += 4) { output[j] = (unsigned char)(input[i] & 0xff); output[j + 1] = (unsigned char)((input[i] >> 8) & 0xff); output[j + 2] = (unsigned char)((input[i] >> 16) & 0xff); output[j + 3] = (unsigned char)((input[i] >> 24) & 0xff); } } /* Decodes input (unsigned char) into output (UINT4). Assumes len is a multiple of 4. */ void MD5::Decode(UINT4* output, const unsigned char* input, unsigned int len) { unsigned int i, j; for (i = 0, j = 0; j < len; i++, j += 4) { output[i] = ((UINT4)input[j]) | (((UINT4)input[j + 1]) << 8) | (((UINT4)input[j + 2]) << 16) | (((UINT4)input[j + 3]) << 24); } } //---------------------------------------------------------------------------------------------------------------------- static const char* hex = "0123456789abcdef"; static std::string toString(unsigned char digest[MD5_DIGEST_LENGTH]) { char x[2 * MD5_DIGEST_LENGTH]; size_t j = 0; for (size_t i = 0; i < MD5_DIGEST_LENGTH; ++i) { x[j++] = hex[(digest[i] & 0xf0) >> 4]; x[j++] = hex[(digest[i] & 0xf)]; } return std::string(x, 2 * MD5_DIGEST_LENGTH); } //---------------------------------------------------------------------------------------------------------------------- MD5::~MD5() {} void MD5::reset() const { Init(&s_); } Hash::digest_t MD5::compute(const void* buffer, long size) { MD5_CTX s; Init(&s); Update(&s, static_cast(buffer), size); unsigned char digest[MD5_DIGEST_LENGTH]; Final(digest, &s); return toString(digest); } MD5::MD5() { Init(&s_); } MD5::MD5(const char* s) { Init(&s_); add(s, strlen(s)); } MD5::MD5(const std::string& s) { Init(&s_); add(s.c_str(), s.size()); } MD5::MD5(const void* data, size_t len) { Init(&s_); add(data, len); } void MD5::update(const void* buffer, long length) { if (length > std::numeric_limits::max()) { throw BadParameter("Buffer length too large for MD5 algorithm", Here()); } if (length > 0) { void* b = const_cast(buffer); MD5::Update(&s_, static_cast(b), length); if (!digest_.empty()) { digest_ = digest_t(); // reset the digest } } } MD5::digest_t MD5::digest() const { // recompute the digest if (digest_.empty()) { unsigned char digest[MD5_DIGEST_LENGTH]; MD5::Final(digest, &s_); digest_ = toString(digest); } return digest_; } void MD5::numericalDigest(unsigned char out[MD5_DIGEST_LENGTH]) const { MD5::Final(&out[0], &s_); } namespace { HashBuilder builder("MD5"); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/HyperCube.cc0000664000175000017500000000375115161702250020233 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/utils/HyperCube.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static void addLoop(Ordinal d, Ordinal which, Ordinal where, Ordinal count, Ordinal depth, HyperCube& target, const HyperCube::Dimensions& dims, HyperCube::Coordinates& coord, HyperCube::Remapping& remap) { if (d == depth) { remap.push_back(target.index(coord)); } else { int k = 0; for (size_t i = 0; i < dims[d]; i++, k++) { if (which == d && i == where) { k += count; } coord[d] = k; addLoop(d + 1, which, where, count, depth, target, dims, coord, remap); } } } HyperCube HyperCube::addToDimension(Ordinal which, Ordinal where, Ordinal howMuch, Remapping& remap) const { remap.clear(); remap.reserve(count()); Dimensions newdims = dimensions_; Coordinates coord(dimensions_.size()); newdims[which] += howMuch; HyperCube target(newdims); addLoop(0, which, where, howMuch, dimensions_.size(), target, dimensions_, coord, remap); return target; } void HyperCube::coordinates(Ordinal index, Coordinates& result) const { ASSERT(result.size() == dimensions_.size()); for (int i = dimensions_.size() - 1; i >= 0; i--) { result[i] = (index % dimensions_[i]); index /= dimensions_[i]; } ASSERT(index == 0); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/LZ4Compressor.cc0000664000175000017500000000434715161702250021035 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/utils/LZ4Compressor.h" #include #include "lz4.h" // header includes extern c linkage #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- LZ4Compressor::LZ4Compressor() {} LZ4Compressor::~LZ4Compressor() {} size_t LZ4Compressor::compress(const void* in, size_t len, Buffer& out) const { ASSERT(len <= std::numeric_limits::max()); ASSERT(out.size() <= std::numeric_limits::max()); const int maxcompressed = LZ4_compressBound(int(len)); if (int(out.size()) < maxcompressed) { out.resize(size_t(maxcompressed)); } const int compressed = LZ4_compress_default(static_cast(in), out, int(len), maxcompressed); if (compressed <= 0) { std::ostringstream msg; msg << "returned " << compressed; throw FailedLibraryCall("LZ4", "LZ4_compress_default", msg.str(), Here()); } return size_t(compressed); } void LZ4Compressor::uncompress(const void* in, size_t len, Buffer& out, size_t outlen) const { if (out.size() < outlen) { out.resize(outlen); } ASSERT(len <= std::numeric_limits::max()); ASSERT(out.size() <= std::numeric_limits::max()); const auto uncompressed = LZ4_decompress_safe(static_cast(in), out, int(len), int(out.size())); if (uncompressed < 0) { std::ostringstream msg; msg << "returned " << uncompressed; throw FailedLibraryCall("LZ4", "LZ4_decompress_safe", msg.str(), Here()); } ASSERT(uncompressed == outlen); } CompressorBuilder lz4("lz4"); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/xxHashing.h0000664000175000017500000000233515161702250020145 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once /// @note This file is named HashxxHash and not simply xxHash not to clash with the included header xxhash.h /// for case insensitive file systems #include #include "eckit/eckit.h" #include "eckit/utils/Hash.h" namespace eckit { class xxHash : public Hash { public: // types xxHash(); explicit xxHash(const char*); explicit xxHash(const std::string&); xxHash(const void* data, size_t len); ~xxHash() override; void reset() const override; digest_t compute(const void*, long) override; void update(const void*, long) override; digest_t digest() const override; template xxHash& operator<<(const T& x) { add(x); return *this; } private: // members struct Context; std::unique_ptr ctx_; }; } // end namespace eckit eckit-2.0.7/src/eckit/utils/RLE.cc0000664000175000017500000002441415161702250016766 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include "eckit/log/Log.h" #include "eckit/serialisation/Stream.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- // Version 2: warning all numbers must be signed but positive... template class dummy_iterator { public: using iterator_category = std::output_iterator_tag; using value_type = T; using difference_type = std::ptrdiff_t; using pointer = value_type*; using reference = value_type&; dummy_iterator() {} dummy_iterator& operator=(const value_type&) { return *this; } dummy_iterator& operator*() { return *this; } dummy_iterator& operator++() { return *this; } dummy_iterator& operator++(int) { return *this; } }; template dummy_iterator make_dummy(T*) { return dummy_iterator(); } template long long RLEencode2timeout(T first, T last, U output, long long maxLoop, std::optional deadline, std::optional maxDepth) { long long x = 0; long long m = 0; long long j = 0; long long size = last - first; if (size <= 0) return 0; long long size2 = (size + 2) / 2; long long enough = std::min(size2, maxLoop); for (long long n = 1; n < size2; n++) { long long sizen = size - n; for (long long i = 0; i < sizen; i += n) { T from = first + i; T other = from + n; long long a = 0; while (from != last && *from == *other) { ++from; ++other; a++; } /* long long a = (mismatch (from, last, from + n).first - from); */ if (a > m) { m = a; x = n; j = i; if (m > enough || (maxDepth && maxDepth.value() == 0) || (deadline && EncodingClock::now() > deadline.value())) goto stop; } } if ((maxDepth && maxDepth.value() == 0) || (deadline && EncodingClock::now() > deadline.value())) { goto stop; } } stop: if (m == 0 || (maxDepth && maxDepth.value() == 0)) { std::copy(first, last, output); return last - first; } else { long long k = 0; T from = first + j; T other = from; while ((other + x <= last) && equal(from, from + x, other)) { k++; other += x; } if (maxDepth) { maxDepth.value()--; } long long n = RLEencode2timeout(first, from, output, maxLoop, deadline, maxDepth); if (k > 1) { *output++ = -k; n++; int m = RLEencode2timeout(from, from + x, make_dummy((typename std::iterator_traits::value_type*)(0)), maxLoop, deadline, maxDepth); if (m > 1) { *output++ = -m; n++; } } n += RLEencode2timeout(from, from + x, output, maxLoop, deadline, maxDepth); n += RLEencode2timeout(from + k * x, last, output, maxLoop, deadline, maxDepth); return n; } } template long long RLEencode2(T first, T last, U output, long long maxLoop) { return RLEencode2timeout(first, last, output, maxLoop, std::optional{}, std::optional{}); } template long long RLEencode2(T first, T last, U output, long long maxLoop, const EncodingClock::duration timelimit, long maxDepth) { return RLEencode2timeout(first, last, output, maxLoop, std::optional{EncodingClock::now() + timelimit}, std::optional{maxDepth}); } template long long RLEencode2(InputIterator first, InputIterator last, OutputIterator result, long long maxloop); template long long RLEencode2(InputIterator first, InputIterator last, OutputIterator result, long long maxloop, const EncodingClock::duration timelimit, size_t maxDepth); template void RLEdecode2(T first, T last, U output) { while (first != last) { if ((long long)*first < 0) { long long repeat = -*first++; if ((long long)*first < 0) { long long length = -*first++; while (repeat--) RLEdecode2(first, first + length, output); first += length; } else { while (repeat--) *output++ = *first; first++; } } else *output++ = *first++; } } template void RLEprint(std::ostream& out, T first, T last) { while (first != last) { if ((long long)*first < 0) { long long repeat = -*first++; if ((long long)*first < 0) { long long length = -*first++; out << repeat << "*("; RLEprint(out, first, first + length); first += length; out << ')'; if (first != last) out << ','; } else { out << repeat << '*' << *first; first++; if (first != last) out << ','; } } else { out << *first++; if (first != last) out << ','; } } } //---------------------------------------------------------------------------------------------------------------------- template bool DIFFencode(InputIterator first, InputIterator last, OutputIterator result) { if (first == last) return true; InputIterator prev = first; *result = *first; ++first; ++result; while (first != last) { if (*first < *prev) return false; *result = (long long)(*first - *prev); // Come back here prev = first; ++first; ++result; } return true; } template void DIFFdecode(InputIterator first, InputIterator last, OutputIterator result, T*) { if (first == last) return; T value = *first; *result = *first; ++first; ++result; while (first != last) { *result = value = *first + value; ++first; ++result; } } template void DIFFdecode(InputIterator first, InputIterator last, OutputIterator result) { DIFFdecode(first, last, result, #if defined(__GNUC__) && __GNUC__ >= 3 (typename InputIterator::value_type*)(0) #else #if defined(VISUAL_AGE) || defined(__hpux) (typename InputIterator::value_type*)(0) #else value_type(first) #endif #endif ); } //---------------------------------------------------------------------------------------------------------------------- template Stream& RLEwrite(Stream& s, InputIterator first, InputIterator last, long long maxLoop, T*) { std::vector tmp; tmp.reserve(last - first); RLEencode2(first, last, std::back_inserter(tmp), maxLoop); s << tmp; Log::info() << "RLEwrite : " << last - first << " -> " << tmp.size() << std::endl; return s; } template Stream& RLEwrite(Stream& s, InputIterator first, InputIterator last, long long maxLoop) { return RLEwrite(s, first, last, maxLoop, #if defined(__GNUC__) && __GNUC__ >= 3 (typename InputIterator::value_type*)(0) #else #if defined(VISUAL_AGE) || defined(__hpux) (typename InputIterator::value_type*)(0) #else value_type(first) #endif #endif ); } template Stream& RLEread(Stream& s, OutputIterator result, T*) { std::vector tmp; s >> tmp; RLEdecode2(tmp.begin(), tmp.end(), result); return s; } template Stream& RLEDIFFwrite(Stream& s, InputIterator first, InputIterator last, long long maxLoop, T*) { std::vector tmp; tmp.reserve(last - first); bool diff = DIFFencode(first, last, std::back_inserter(tmp)); s << diff; if (!diff) { // Warning, does not work with ostream iterator, // as we reuse first, last Log::warning() << "DIFF encoding failed." << std::endl; tmp.clear(); std::copy(first, last, std::back_inserter(tmp)); s << tmp; } else { return RLEwrite(s, tmp.begin(), tmp.end(), maxLoop); } return s; } template Stream& RLEDIFFwrite(Stream& s, InputIterator first, InputIterator last, long long maxLoop) { return RLEDIFFwrite(s, first, last, maxLoop, #if defined(__GNUC__) && __GNUC__ >= 3 (typename InputIterator::value_type*)(0) #else #if defined(VISUAL_AGE) || defined(__hpux) (typename InputIterator::value_type*)(0) #else value_type(first) #endif #endif ); } template Stream& RLEDIFFread(Stream& s, OutputIterator result, T* dummy) { bool diff; s >> diff; if (diff) { std::vector tmp; RLEread(s, std::back_inserter(tmp), dummy); DIFFdecode(tmp.begin(), tmp.end(), result); } else { std::vector tmp; s >> tmp; std::copy(tmp.begin(), tmp.end(), result); } return s; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/Tokenizer.cc0000664000175000017500000000530115161702250020310 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "Tokenizer.h" #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template void tokenizeInsert(const std::set >& separator, const std::string& raw, std::insert_iterator ins, bool keepEmpty) { int index = 0; int length = raw.length(); std::string token = ""; while (index < length) { char c = raw[index]; if (separator.find(c) != separator.end()) { if (token.length() > 0 || keepEmpty) { ins = token; } token = ""; } else { token += c; } index++; } if (token.length() > 0 || keepEmpty) { ins = token; } } //---------------------------------------------------------------------------------------------------------------------- Tokenizer::Tokenizer(char c, bool keepEmpty) : keepEmpty_(keepEmpty) { separator_.insert(c); } Tokenizer::Tokenizer(const std::string& separators, bool keepEmpty) : keepEmpty_(keepEmpty) { for (std::string::size_type i = 0; i < separators.length(); i++) { separator_.insert(separators[i]); } } Tokenizer::~Tokenizer() {} void Tokenizer::operator()(const std::string& raw, std::vector& v) const { tokenizeInsert(separator_, raw, std::inserter(v, v.end()), keepEmpty_); } void Tokenizer::operator()(std::istream& in, std::vector& v) const { std::string raw; char c; while (in.get(c) && c != EOF && c != '\n') { raw += c; } tokenizeInsert(separator_, raw, std::inserter(v, v.end()), keepEmpty_); } void Tokenizer::operator()(const std::string& raw, std::set& s) const { tokenizeInsert(separator_, raw, std::inserter(s, s.end()), keepEmpty_); } void Tokenizer::operator()(std::istream& in, std::set& s) const { std::string raw; char c; while (in.get(c) && c != EOF && c != '\n') { raw += c; } tokenizeInsert(separator_, raw, std::inserter(s, s.end()), keepEmpty_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/Compressor.cc0000664000175000017500000000756715161702250020512 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/utils/Compressor.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/thread/AutoLock.h" #include "eckit/utils/StringTools.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- CompressorFactory::CompressorFactory() = default; CompressorFactory& CompressorFactory::instance() { static CompressorFactory theOne; return theOne; } void CompressorFactory::add(const std::string& name, CompressorBuilderBase* builder) { AutoLock lock(mutex_); auto key = StringTools::lower(name); if (has(key)) { throw SeriousBug("Duplicate entry in CompressorFactory: " + key, Here()); } builders_[key] = builder; } void CompressorFactory::remove(const std::string& name) { AutoLock lock(mutex_); auto key = StringTools::lower(name); builders_.erase(key); } bool CompressorFactory::has(const std::string& name) { AutoLock lock(mutex_); auto key = StringTools::lower(name); return builders_.find(key) != builders_.end(); } std::vector CompressorFactory::keys() const { AutoLock lock(mutex_); std::vector keys; for (const auto& builder : builders_) { keys.push_back(builder.first); } return keys; } void CompressorFactory::list(std::ostream& out) { AutoLock lock(mutex_); const auto* sep = ""; for (const auto& j : builders_) { out << sep << j.first; sep = ", "; } } Compressor* CompressorFactory::build() { std::string compression = Resource("defaultCompression;ECKIT_DEFAULT_COMPRESSION", "snappy"); return build(has(compression) ? compression : "none"); } Compressor* CompressorFactory::build(const std::string& name) { AutoLock lock(mutex_); auto key = StringTools::lower(name); if (auto j = builders_.find(key); j != builders_.end()) { return (*j).second->make(); } list(Log::error() << "No CompressorBuilder for [" << key << "]. CompressorBuilders are:"); Log::error() << std::endl; throw SeriousBug(std::string("No CompressorBuilder called ") + key); } //---------------------------------------------------------------------------------------------------------------------- CompressorBuilderBase::CompressorBuilderBase(const std::string& name) : name_(name) { CompressorFactory::instance().add(name_, this); } CompressorBuilderBase::~CompressorBuilderBase() { CompressorFactory::instance().remove(name_); } //---------------------------------------------------------------------------------------------------------------------- NoCompressor::NoCompressor() = default; size_t NoCompressor::compress(const void* in, size_t len, Buffer& out) const { if (out.size() < len) { out.resize(len); } out.copy(in, len); return len; } void NoCompressor::uncompress(const void* in, size_t len, Buffer& out, size_t outlen) const { ASSERT(outlen == len); if (out.size() < outlen) { out.resize(outlen); } out.copy(in, len); } //---------------------------------------------------------------------------------------------------------------------- namespace { CompressorBuilder builder1("none"); } // namespace //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/AECCompressor.h0000664000175000017500000000214715161702250020652 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_AECCompressor_H #define eckit_utils_AECCompressor_H #include "eckit/utils/Compressor.h" namespace eckit { class Buffer; //---------------------------------------------------------------------------------------------------------------------- class AECCompressor : public eckit::Compressor { public: // methods AECCompressor(); ~AECCompressor() override; size_t compress(const void* in, size_t len, eckit::Buffer& out) const override; void uncompress(const void* in, size_t len, eckit::Buffer& out, size_t outlen) const override; }; //---------------------------------------------------------------------------------------------------------------------- } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/MD5.h0000664000175000017500000000610715161702250016572 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_MD5_H #define eckit_utils_MD5_H #ifndef MD5_DIGEST_LENGTH #define MD5_DIGEST_LENGTH 16 #endif #include "eckit/utils/Hash.h" namespace eckit { class MD5 : public Hash { public: // types MD5(); explicit MD5(const char*); explicit MD5(const std::string&); MD5(const void* data, size_t len); ~MD5() override; void reset() const override; digest_t compute(const void*, long) override; void update(const void*, long) override; digest_t digest() const override; // Due to C++ name lookup rules, the base class interface needs to be // replicated here to be able to add the template overload, which would // otherwise shadow those methods. void add(char x) { update(&x, sizeof(x)); } void add(unsigned char x) { update(&x, sizeof(x)); } void add(bool x) { update(&x, sizeof(x)); } void add(int x) { update(&x, sizeof(x)); } void add(unsigned int x) { update(&x, sizeof(x)); } void add(short x) { update(&x, sizeof(x)); } void add(unsigned short x) { update(&x, sizeof(x)); } void add(long x) { update(&x, sizeof(x)); } void add(unsigned long x) { update(&x, sizeof(x)); } void add(long long x) { update(&x, sizeof(x)); } void add(unsigned long long x) { update(&x, sizeof(x)); } void add(float x) { update(&x, sizeof(x)); } void add(double x) { update(&x, sizeof(x)); } void add(const void* x, long size) { update(x, size); } void add(const std::string& x) { update(x.c_str(), x.size()); } void add(const char* x) { update(x, std::strlen(x)); } // for generic objects template void add(const T& x) { x.hash(*this); } template MD5& operator<<(const T& x) { add(x); return *this; } void numericalDigest(unsigned char out[MD5_DIGEST_LENGTH]) const; private: // members mutable digest_t digest_; ///< cached digest /* POINTER defines a generic pointer type */ using POINTER = unsigned char*; /* UINT4 defines a four byte word */ using UINT4 = uint32_t; /* MD5 context. */ struct MD5_CTX { UINT4 state[4]; /* state (ABCD) */ UINT4 count[2]; /* number of bits, modulo 2^64 (lsb first) */ unsigned char buffer[64]; /* input buffer */ }; mutable MD5_CTX s_; static void Init(MD5_CTX*); static void Update(MD5_CTX*, const unsigned char*, unsigned int); static void Final(unsigned char[16], MD5_CTX*); static void Transform(UINT4[4], const unsigned char[64]); static void Encode(unsigned char*, UINT4*, unsigned int); static void Decode(UINT4*, const unsigned char*, unsigned int); }; } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/Hash.cc0000664000175000017500000001137415161702250017230 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/utils/Hash.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" #include "eckit/utils/StringTools.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- HashFactory::HashFactory() {} HashFactory& HashFactory::instance() { static HashFactory theOne; return theOne; } void HashFactory::add(const std::string& name, HashBuilderBase* builder) { std::string nameLowercase = StringTools::lower(name); AutoLock lock(mutex_); if (has(nameLowercase)) { throw SeriousBug("Duplicate entry in HashFactory: " + nameLowercase, Here()); } builders_[nameLowercase] = builder; } void HashFactory::remove(const std::string& name) { std::string nameLowercase = StringTools::lower(name); AutoLock lock(mutex_); builders_.erase(nameLowercase); } bool HashFactory::has(const std::string& name) { std::string nameLowercase = StringTools::lower(name); AutoLock lock(mutex_); return builders_.find(nameLowercase) != builders_.end(); } void HashFactory::list(std::ostream& out) { AutoLock lock(mutex_); const char* sep = ""; for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); ++j) { out << sep << (*j).first; sep = ", "; } } Hash* HashFactory::build() { std::string name = eckit::Resource("defaultHash;ECKIT_DEFAULT_HASH", "md5"); if (has(name)) { return build(name); } return build("none"); } Hash* HashFactory::build(const std::string& name) { std::string nameLowercase = StringTools::lower(name); AutoLock lock(mutex_); auto j = builders_.find(nameLowercase); eckit::Log::debug() << "Looking for HashBuilder [" << nameLowercase << "]" << std::endl; if (j == builders_.end()) { eckit::Log::error() << "No HashBuilder for [" << nameLowercase << "]" << std::endl; eckit::Log::error() << "HashBuilders are:" << std::endl; for (j = builders_.begin(); j != builders_.end(); ++j) { eckit::Log::error() << " " << (*j).first << std::endl; } throw eckit::SeriousBug(std::string("No HashBuilder called ") + nameLowercase); } return (*j).second->make(); } Hash* HashFactory::build(const std::string& name, const std::string& param) { std::string nameLowercase = StringTools::lower(name); AutoLock lock(mutex_); auto j = builders_.find(nameLowercase); eckit::Log::debug() << "Looking for HashBuilder [" << nameLowercase << "]" << std::endl; if (j == builders_.end()) { eckit::Log::error() << "No HashBuilder for [" << nameLowercase << "]" << std::endl; eckit::Log::error() << "HashBuilders are:" << std::endl; for (j = builders_.begin(); j != builders_.end(); ++j) { eckit::Log::error() << " " << (*j).first << std::endl; } throw eckit::SeriousBug(std::string("No HashBuilder called ") + nameLowercase); } return (*j).second->make(param); } //---------------------------------------------------------------------------------------------------------------------- HashBuilderBase::HashBuilderBase(const std::string& name) : name_(name) { HashFactory::instance().add(name_, this); } HashBuilderBase::~HashBuilderBase() { HashFactory::instance().remove(name_); } //---------------------------------------------------------------------------------------------------------------------- Hash::Hash() {} Hash::~Hash() {} //---------------------------------------------------------------------------------------------------------------------- class NoHash : public Hash { public: // types NoHash() {} NoHash(const std::string&) {} virtual ~NoHash() {} virtual void reset() const {} virtual digest_t compute(const void*, long) { return std::string(); } virtual void update(const void*, long) {} virtual digest_t digest() const { return digest_; // should be empty } }; namespace { HashBuilder builder1("None"); HashBuilder builder2("NoHash"); } // namespace //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/EnumBitmask.h0000664000175000017500000000564215161702250020427 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_EnumBitmasks_H #define eckit_utils_EnumBitmasks_H namespace eckit { #include #define ENUM_FLAG_OPERATORS(T) \ inline constexpr T operator&(T X, T Y) { \ return static_cast(static_cast::type>(X) & \ static_cast::type>(Y)); \ } \ inline constexpr T operator|(T X, T Y) { \ return static_cast(static_cast::type>(X) | \ static_cast::type>(Y)); \ } \ inline constexpr T operator^(T X, T Y) { \ return static_cast(static_cast::type>(X) ^ \ static_cast::type>(Y)); \ } \ inline constexpr T operator~(T X) { \ return static_cast(~static_cast::type>(X)); \ } \ inline T& operator&=(T& X, T Y) { \ X = X & Y; \ return X; \ } \ inline T& operator|=(T& X, T Y) { \ X = X | Y; \ return X; \ } \ inline T& operator^=(T& X, T Y) { \ X = X ^ Y; \ return X; \ } } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/Literals.h0000664000175000017500000000275615161702250017772 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include /// Eckit literals provides user defined literals for IEC multi-byte units. /// To make these literals available to your code use `using namespace eckit::literals` /// Usage example: /// ``` /// using namespace eckit::literals; /// const auto bufferSize = 3_MiB; /// ``` namespace eckit::literals { /// Literal to express Kibibyte constexpr std::uint64_t operator""_KiB(unsigned long long int x) { return 1024ULL * x; } /// Literal to express Mebibyte constexpr std::uint64_t operator""_MiB(unsigned long long int x) { return 1024_KiB * x; } /// Literal to express Gibibyte constexpr std::uint64_t operator""_GiB(unsigned long long int x) { return 1024_MiB * x; } /// Literal to express Tebibyte constexpr std::uint64_t operator""_TiB(unsigned long long int x) { return 1024_GiB * x; } /// Literal to express Pebibyte constexpr std::uint64_t operator""_PiB(unsigned long long int x) { return 1024_TiB * x; } /// Literal to express Exbibyte constexpr std::uint64_t operator""_EiB(unsigned long long int x) { return 1024_PiB * x; } } // namespace eckit::literals eckit-2.0.7/src/eckit/utils/SafeCasts.h0000664000175000017500000000464415161702250020065 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /** * Place functions to allow for save type casting in this file */ #pragma once #include #include #include "eckit/exception/Exceptions.h" namespace eckit { /** * Casts signed integer into unsigned. * @param value to cast from. * @return value cast into, same as before but unsigned type. * @throws BadCast if used with a negative value. */ template && std::is_signed_v, int> = 0> [[nodiscard]] constexpr auto into_unsigned(S value) -> std::make_unsigned_t { using U = std::make_unsigned_t; if (value < 0) { throw eckit::BadCast("Negative value cannot be cast to unsigned type", Here()); } return static_cast(value); } /** * Template specialization that turns unsigned to unsigned conversions with 'into_unsigned' into a NoOp * @param value * @return value, returned unmodified. */ template && std::is_unsigned_v, int> = 0> [[nodiscard]] constexpr auto into_unsigned(S value) -> std::make_unsigned_t { return value; } /** * Casts unsigned integer into signed. * @param value to cast from. * @return value cast into, same as before but signed type. * @throws BadCast if used with a value > 2^(bits-1)-1 */ template && std::is_unsigned_v, int> = 0> [[nodiscard]] constexpr auto into_signed(U value) -> std::make_signed_t { using S = std::make_signed_t; if (value > static_cast(std::numeric_limits::max())) { throw eckit::BadCast("Value too large to cast to signed type"); } return static_cast(value); } /** * Template specialization that turns signed to signed conversions with 'into_signed' into a NoOp * @param value * @return value, returned unmodified. */ template && std::is_signed_v, int> = 0> [[nodiscard]] constexpr auto into_signed(U value) -> std::make_signed_t { return value; } } // namespace eckit eckit-2.0.7/src/eckit/utils/xxHashing.cc0000664000175000017500000000556615161702250020314 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #define XXH_INLINE_ALL #include "eckit/contrib/xxhash/xxhash.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/xxHashing.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- struct xxHash::Context { XXH64_state_t* state_; Context() { state_ = XXH64_createState(); reset(); } ~Context() { XXH64_freeState(state_); } void reset() { XXH64_reset(state_, 0); } void update(const void* buffer, long length) { XXH64_update(state_, buffer, size_t(length)); } std::string digest() { return toString(XXH64_digest(state_)); } static std::string compute(const void* buffer, long length) { return toString(XXH64(buffer, size_t(length), 0)); } static std::string toString(XXH64_hash_t hash) { static const char* hex = "0123456789abcdef"; char buffer[16]; for (int i = 16; i--;) { buffer[i] = hex[hash & 15]; hash >>= 4; } return std::string(buffer, buffer + 16); } }; //---------------------------------------------------------------------------------------------------------------------- xxHash::xxHash() { ctx_.reset(new Context()); } xxHash::xxHash(const char* s) { ctx_.reset(new Context()); add(s, strlen(s)); } xxHash::xxHash(const std::string& s) { ctx_.reset(new Context()); add(s.c_str(), s.size()); } xxHash::xxHash(const void* data, size_t len) { ctx_.reset(new Context()); add(data, len); } xxHash::~xxHash() {} void xxHash::reset() const { ctx_->reset(); } Hash::digest_t xxHash::compute(const void* buffer, long size) { return Context::compute(buffer, size); } void xxHash::update(const void* buffer, long length) { if (length > 0) { ctx_->update(buffer, length); if (!digest_.empty()) { digest_ = digest_t(); // reset the digest } } } xxHash::digest_t xxHash::digest() const { if (digest_.empty()) { // recompute the digest digest_ = ctx_->digest(); } return digest_; } //---------------------------------------------------------------------------------------------------------------------- namespace { HashBuilder deprecated_builder("xxHash"); HashBuilder builder("xxh64"); } // namespace //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/Overloaded.h0000664000175000017500000000154715161702250020274 0ustar alastairalastair#pragma once //----------------------------------------------------------------------------- namespace eckit { // Overload pattern for visiting std::variant using std::visit, see // https://en.cppreference.com/w/cpp/utility/variant/visit // The struct Overloaded can have arbitrary many base classes (Ts ...). It publicly inherits from each class and brings // the call operator (Ts::operator...) of each base class into its scope. The base classes need an overloaded call // operator (Ts::operator()). // Overload pattern template struct Overloaded : Ts... { using Ts::operator()...; }; // Explicit deduction guide for the overload pattern above (not needed as of C++20) template Overloaded(Ts...) -> Overloaded; } // namespace eckit //----------------------------------------------------------------------------- eckit-2.0.7/src/eckit/utils/MD4.cc0000664000175000017500000000452115161702250016725 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/utils/MD4.h" // Cray C++ compiler should not try to optimize this code #if _CRAYC #pragma _CRI noopt #endif namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static const char* hex = "0123456789abcdef"; static std::string toString(unsigned char digest[MD4_DIGEST_LENGTH]) { char x[2 * MD4_DIGEST_LENGTH]; size_t j = 0; for (size_t i = 0; i < MD4_DIGEST_LENGTH; ++i) { x[j++] = hex[(digest[i] & 0xf0) >> 4]; x[j++] = hex[(digest[i] & 0xf)]; } return std::string(x, 2 * MD4_DIGEST_LENGTH); } //---------------------------------------------------------------------------------------------------------------------- MD4::~MD4() {} void MD4::reset() const { MD4_Init(&ctx_); } Hash::digest_t MD4::compute(const void* buffer, long size) { MD4_CTX s; MD4_Init(&s); MD4_Update(&s, static_cast(buffer), size); unsigned char digest[MD4_DIGEST_LENGTH]; MD4_Final(digest, &s); return toString(digest); } MD4::MD4() { MD4_Init(&ctx_); } MD4::MD4(const char* s) { MD4_Init(&ctx_); add(s, strlen(s)); } MD4::MD4(const std::string& s) { MD4_Init(&ctx_); add(s.c_str(), s.size()); } MD4::MD4(const void* data, size_t len) { MD4_Init(&ctx_); add(data, len); } void MD4::update(const void* buffer, long length) { if (length > 0) { MD4_Update(&ctx_, static_cast(buffer), length); if (!digest_.empty()) digest_ = digest_t(); // reset the digest } } MD4::digest_t MD4::digest() const { if (digest_.empty()) { // recompute the digests unsigned char digest[MD4_DIGEST_LENGTH]; MD4_Final(digest, &ctx_); digest_ = toString(digest); } return digest_; } namespace { HashBuilder builder("MD4"); } } // namespace eckit eckit-2.0.7/src/eckit/utils/SnappyCompressor.h0000664000175000017500000000221615161702250021531 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_SnappyCompressor_H #define eckit_utils_SnappyCompressor_H #include "eckit/utils/Compressor.h" namespace eckit { class Buffer; //---------------------------------------------------------------------------------------------------------------------- class SnappyCompressor : public eckit::Compressor { public: // methods SnappyCompressor(); ~SnappyCompressor() override; size_t compress(const void* in, size_t len, eckit::Buffer& out) const override; void uncompress(const void* in, size_t len, eckit::Buffer& out, size_t outlen) const override; protected: // methods }; //---------------------------------------------------------------------------------------------------------------------- } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/RendezvousHash.h0000664000175000017500000000524315161702250021155 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_RendezvousHash_H #define eckit_utils_RendezvousHash_H #include #include #include #include "eckit/thread/Mutex.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// This class implements the Rendezvous or Highest Random Weight (HRW) hashing /// It is thread-safe, in terms that threads can add and remove nodes whilts others can /// compute the hash and obtain the rendezvous node /// /// @todo Make node a template parameter? Must be a serializable object class RendezvousHash { public: // types using Node = std::string; using Key = std::map; using hash_func_ptr = std::string (*)(const std::string&); private: // types using iterator = std::vector::iterator; public: // methods static std::string md5(const std::string& str); RendezvousHash(const hash_func_ptr hash = &md5); RendezvousHash(const std::vector& nodes, const hash_func_ptr hash = &md5); RendezvousHash(const RendezvousHash&) = delete; RendezvousHash& operator=(const RendezvousHash&) = delete; RendezvousHash(RendezvousHash&&) = delete; RendezvousHash& operator=(RendezvousHash&&) = delete; ~RendezvousHash(); /// Provide a list of nodes / indices in the list of nodes for the given key void hashOrder(const Key& key, std::vector& nodes); void hashOrder(const Key& key, std::vector& indices); /// Adds node to node list. No effect if node already present /// @returns true is node insertion was successful bool addNode(const Node& node); /// Removes node from node list. No effect if node not present /// @returns true is node removal was successful bool removeNode(const Node& node); private: // methods std::string flatten(const Key&) const; void hashOrderInternal(const Key& key, std::vector& indexes); private: // types eckit::Mutex mutex_; //< protects addition and removal of nodes hash_func_ptr hash_; std::vector nodes_; }; //---------------------------------------------------------------------------------------------------------------------- } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/ByteSwap.h0000664000175000017500000000732015161702250017741 0ustar alastairalastair/* * (C) Copyright 2020- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @date October 2020 #ifndef eckit_utils_ByteSwap_H #define eckit_utils_ByteSwap_H #include #include #include #include #include #include "eckit/eckit.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- // Low Level bitswap functions inline uint16_t bitswap16(uint16_t a) { a = ((a & 0x00FF) << 8) | ((a & 0xFF00) >> 8); return a; } inline uint32_t bitswap32(uint32_t a) { a = ((a & 0x000000FF) << 24) | ((a & 0x0000FF00) << 8) | ((a & 0x00FF0000) >> 8) | ((a & 0xFF000000) >> 24); return a; } inline uint64_t bitswap64(uint64_t a) { a = ((a & 0x00000000000000FFULL) << 56) | ((a & 0x000000000000FF00ULL) << 40) | ((a & 0x0000000000FF0000ULL) << 24) | ((a & 0x00000000FF000000ULL) << 8) | ((a & 0x000000FF00000000ULL) >> 8) | ((a & 0x0000FF0000000000ULL) >> 24) | ((a & 0x00FF000000000000ULL) >> 40) | ((a & 0xFF00000000000000ULL) >> 56); return a; } //---------------------------------------------------------------------------------------------------------------------- /// ByteSwap selects the swap function based on data type size template struct ByteSwap {}; template <> struct ByteSwap<8> { using inter_t = uint64_t; static void bitswap(uint64_t& i) { i = eckit::bitswap64(i); } }; template <> struct ByteSwap<4> { using inter_t = uint32_t; static void bitswap(uint32_t& i) { i = eckit::bitswap32(i); } }; template <> struct ByteSwap<2> { using inter_t = uint16_t; static void bitswap(uint16_t& i) { i = eckit::bitswap16(i); } }; //---------------------------------------------------------------------------------------------------------------------- /// Scalar bitswap template function uses ByteSwap to select based on type T size template void byteswap(T& i) { constexpr std::size_t sz = sizeof(T); // std::cerr << "byteswap(T) sizeof T = " << sz << std::endl; typename ByteSwap::inter_t* v = reinterpret_cast::inter_t*>(&i); ByteSwap::bitswap(*v); } /// C-array bitswap template function uses ByteSwap to select based on type T size template void byteswap(T data[], size_t size) { constexpr std::size_t sz = sizeof(T); // std::cerr << "array byteswap(T) sizeof T = " << sz << std::endl; auto vt = reinterpret_cast::inter_t*>(data); std::for_each(vt, vt + size, [](typename ByteSwap::inter_t& e) { eckit::byteswap(e); }); } /// std::vector bitswap template function uses ByteSwap to select based on type T size template void byteswap(std::vector& vi) { eckit::byteswap(vi.data(), vi.size()); } //---------------------------------------------------------------------------------------------------------------------- template std::string bits_to_str(T v) { constexpr std::size_t sz = sizeof(T); typename ByteSwap::inter_t* t = reinterpret_cast::inter_t*>(&v); constexpr std::size_t nbits = 8 * sizeof(T); std::bitset bits(*t); return bits.to_string(); } //---------------------------------------------------------------------------------------------------------------------- } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/LZ4Compressor.h0000664000175000017500000000217715161702250020676 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_LZ4Compressor_H #define eckit_utils_LZ4Compressor_H #include "eckit/utils/Compressor.h" namespace eckit { class Buffer; //---------------------------------------------------------------------------------------------------------------------- class LZ4Compressor : public eckit::Compressor { public: // methods LZ4Compressor(); ~LZ4Compressor() override; size_t compress(const void* in, size_t len, eckit::Buffer& out) const override; void uncompress(const void* in, size_t len, eckit::Buffer& out, size_t outlen) const override; protected: // methods }; //---------------------------------------------------------------------------------------------------------------------- } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/Translator.h0000664000175000017500000001441215161702250020334 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Translator.h /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Jun 1996 #ifndef eckit_Translator_h #define eckit_Translator_h #include #include #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- // Translate a type from From to To template struct Translator { // Test for explicit conversion through constructor (also involves implicit conversion or or user-defined conversion // operator). Note: To(from) is called with () brackets and not with {} because it includes conversion of plain // datatypes - MIR is using this template , To> && std::is_constructible_v), bool> = true> auto operator()(F&& from) { return To(std::forward(from)); } // If from and to type are same - simply forward - i.e. allow moving or passing references instead of performing // copies template , To>, bool> = true> decltype(auto) operator()(F&& from) { if constexpr (std::is_lvalue_reference_v && !std::is_const_v) { return const_cast(from); } else { return std::forward(from); } } }; // Those are predefined template <> struct Translator { std::string operator()(bool); }; template <> struct Translator { bool operator()(const std::string&); }; template <> struct Translator { std::string operator()(unsigned char); }; template <> struct Translator { std::string operator()(float); }; template <> struct Translator { std::string operator()(short); }; template <> struct Translator { std::string operator()(int); }; template <> struct Translator { std::string operator()(unsigned int); }; template <> struct Translator { int operator()(const std::string&); }; template <> struct Translator { unsigned int operator()(const std::string&); }; template <> struct Translator { std::string operator()(double); }; template <> struct Translator { double operator()(const std::string&); }; template <> struct Translator { float operator()(const std::string&); }; template <> struct Translator { std::string operator()(long); }; template <> struct Translator { long operator()(const std::string&); }; template <> struct Translator { short operator()(const std::string&); }; template <> struct Translator { std::string operator()(unsigned long); }; template <> struct Translator { unsigned long operator()(const std::string&); }; template <> struct Translator { unsigned char operator()(const std::string&); }; template <> struct Translator { unsigned long long operator()(const std::string&); }; template <> struct Translator { long long operator()(const std::string&); }; template <> struct Translator { std::string operator()(unsigned long long); }; template <> struct Translator { std::string operator()(long long); }; template <> struct Translator { char operator()(const std::string&); }; template <> struct Translator { std::string operator()(char); }; template <> struct Translator> { std::vector operator()(const std::string&); }; template <> struct Translator> { std::vector operator()(const std::string&); }; template <> struct Translator, std::string> { std::string operator()(const std::vector&); }; template <> struct Translator, std::string> { std::string operator()(const std::vector&); }; template <> struct Translator> { std::set operator()(const std::string&); }; template <> struct Translator, std::string> { std::string operator()(const std::set&); }; template <> struct Translator { std::string operator()(signed char v); }; //---------------------------------------------------------------------------------------------------------------------- // This allows using the Translator without having to explicitly name the type of an argument. For example in case of // generic string conversion: translate(someVariable) template decltype(auto) translate(From&& from) { return Translator, To>{}(std::forward(from)); } //---------------------------------------------------------------------------------------------------------------------- // primary template handles types that do not support translation template struct IsTranslatable : std::false_type {}; // specialization recognizes types that do support translation template struct IsTranslatable{}(std::declval()))>> : std::true_type {}; template inline constexpr bool IsTranslatable_v = IsTranslatable::value; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/utils/Compressor.h0000664000175000017500000001106015161702250020333 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_Compressor_H #define eckit_utils_Compressor_H #include #include #include #include "eckit/thread/Mutex.h" namespace eckit { class Buffer; //---------------------------------------------------------------------------------------------------------------------- class Compressor { public: // methods Compressor() = default; Compressor(const Compressor&) = delete; Compressor(Compressor&&) = delete; void operator=(const Compressor&) = delete; void operator=(Compressor&&) = delete; virtual ~Compressor() = default; /// Compresses the bytestream within the in buffer /// @param in input buffer that holds the uncompressed bytesteam. /// @param len input buffer size. /// @param out output buffer to hold the compressed bytesteam. Buffer may be oversized, in which case /// resizing is implementation specific. Nevertheless, it is expected that implementations /// will take this as oppurtunity for optimising and avoid resizing. /// @note if needed, it will resize or replace the internal buffer of out to match the compressed bytestream size /// @returns the size of the compressed bytestream inside out buffer, it is less or equal than out.size() virtual size_t compress(const void* in, size_t len, eckit::Buffer& out) const = 0; /// Uncompresses the bytestream within the in buffer /// Since some compression algorithms dont record uncompressed size in the bytestream, client code is /// required to record and handle the uncompressed size separately and pass it in via parameter outlen. /// @param in input buffer that holds the compressed bytesteam. /// @param len input buffer size. /// @param out output buffer to hold the uncompressed bytesteam. /// @param outlen the expected size of the uncompressed bytestream. Buffer may be resized or if large enough /// implementations will try to used what is passed in. This could be a way to reuse the same memory /// buffer in a tight loop. /// @note may resize or replace the internal buffer of out virtual void uncompress(const void* in, size_t len, eckit::Buffer& out, size_t outlen) const = 0; }; //---------------------------------------------------------------------------------------------------------------------- class NoCompressor : public Compressor { public: // types NoCompressor(); size_t compress(const void* in, size_t len, eckit::Buffer& out) const override; void uncompress(const void* in, size_t len, eckit::Buffer& out, size_t outlen) const override; }; //---------------------------------------------------------------------------------------------------------------------- class CompressorBuilderBase { std::string name_; public: explicit CompressorBuilderBase(const std::string&); CompressorBuilderBase(const CompressorBuilderBase&) = delete; CompressorBuilderBase(CompressorBuilderBase&&) = delete; void operator=(const CompressorBuilderBase&) = delete; void operator=(CompressorBuilderBase&&) = delete; virtual ~CompressorBuilderBase(); virtual Compressor* make() = 0; }; template class CompressorBuilder : public CompressorBuilderBase { Compressor* make() override { return new T(); } public: explicit CompressorBuilder(const std::string& name) : CompressorBuilderBase(name) {} }; class CompressorFactory { public: static CompressorFactory& instance(); void add(const std::string& name, CompressorBuilderBase* builder); void remove(const std::string& name); bool has(const std::string& name); std::vector keys() const; void list(std::ostream&); /// @returns default compressor Compressor* build(); /** * @param name compressor name * @returns compressor built by specified builder */ Compressor* build(const std::string&); private: CompressorFactory(); std::map builders_; mutable eckit::Mutex mutex_; }; //---------------------------------------------------------------------------------------------------------------------- } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/SHA1.cc0000664000175000017500000000362415161702250017040 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/utils/SHA1.h" // Cray C++ compiler should not try to optimize this code #if _CRAYC #pragma _CRI noopt #endif namespace eckit { SHA1::SHA1() { SHA1_Init(&ctx_); } SHA1::SHA1(const char* s) { SHA1_Init(&ctx_); add(s, strlen(s)); } SHA1::SHA1(const std::string& s) { SHA1_Init(&ctx_); add(s.c_str(), s.size()); } SHA1::SHA1(const void* data, size_t len) { SHA1_Init(&ctx_); add(data, len); } SHA1::~SHA1() {} void SHA1::reset() const { SHA1_Init(&ctx_); } Hash::digest_t SHA1::compute(const void* buffer, long size) { SHA1 hash(buffer, size); return hash.digest(); } void SHA1::update(const void* buffer, long length) { if (length > 0) { SHA1_Update(&ctx_, static_cast(buffer), length); if (!digest_.empty()) digest_ = digest_t(); // reset the digest } } static const char* hex = "0123456789abcdef"; SHA1::digest_t SHA1::digest() const { if (digest_.empty()) { // recompute the digest unsigned char digest[SHA_DIGEST_LENGTH]; SHA1_Final(digest, &ctx_); char x[2 * SHA_DIGEST_LENGTH]; size_t j = 0; for (size_t i = 0; i < SHA_DIGEST_LENGTH; ++i) { x[j++] = hex[(digest[i] & 0xf0) >> 4]; x[j++] = hex[(digest[i] & 0xf)]; } digest_ = std::string(x, 2 * SHA_DIGEST_LENGTH); } return digest_; } namespace { HashBuilder builder("SHA1"); } } // namespace eckit eckit-2.0.7/src/eckit/utils/BZip2Compressor.h0000664000175000017500000000221115161702250021200 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_BZip2Compressor_H #define eckit_utils_BZip2Compressor_H #include "eckit/utils/Compressor.h" namespace eckit { class Buffer; //---------------------------------------------------------------------------------------------------------------------- class BZip2Compressor : public eckit::Compressor { public: // methods BZip2Compressor(); ~BZip2Compressor() override; size_t compress(const void* in, size_t len, eckit::Buffer& out) const override; void uncompress(const void* in, size_t len, eckit::Buffer& out, size_t outlen) const override; protected: // methods }; //---------------------------------------------------------------------------------------------------------------------- } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/RLE.h0000664000175000017500000000435515161702250016632 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File RLE.h // Baudouin Raoult - ECMWF Jun 96 #ifndef eckit_RLE_h #define eckit_RLE_h #include #include //----------------------------------------------------------------------------- namespace eckit { using EncodingClock = std::chrono::steady_clock; //----------------------------------------------------------------------------- class Stream; template long long RLEencode2(InputIterator first, InputIterator last, OutputIterator result, long long maxLoop); template long long RLEencode2(InputIterator first, InputIterator last, OutputIterator result, long long maxLoop, const EncodingClock::duration timeLimit, size_t maxDepth = 1000); template void RLEdecode2(InputIterator first, InputIterator last, OutputIterator result); template void RLEprint(std::ostream&, InputIterator first, InputIterator last); template bool DIFFencode(InputIterator first, InputIterator last, OutputIterator result); template void DIFFdecode(InputIterator first, InputIterator last, OutputIterator result); //========================================================================== template Stream& RLEwrite(Stream&, InputIterator, InputIterator, long long); template Stream& RLEread(Stream&, OutputIterator, T*); template Stream& RLEDIFFwrite(Stream&, InputIterator, InputIterator, long long); template Stream& RLEDIFFread(Stream&, OutputIterator, T*); //----------------------------------------------------------------------------- } // namespace eckit #include "eckit/utils/RLE.cc" #endif eckit-2.0.7/src/eckit/utils/RendezvousHash.cc0000664000175000017500000000734015161702250021313 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/utils/RendezvousHash.h" #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/thread/AutoLock.h" #include "eckit/types/Types.h" #include "eckit/utils/MD5.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- std::string RendezvousHash::md5(const std::string& str) { eckit::MD5 md5(str.c_str(), str.size()); return md5.digest(); } RendezvousHash::RendezvousHash(const RendezvousHash::hash_func_ptr hash) : hash_(hash) {} RendezvousHash::RendezvousHash(const std::vector& nodes, const RendezvousHash::hash_func_ptr hash) : hash_(hash), nodes_(nodes) {} RendezvousHash::~RendezvousHash() {} void RendezvousHash::hashOrder(const RendezvousHash::Key& key, std::vector& nodes) { std::vector indexes; AutoLock lock(mutex_); hashOrder(key, indexes); ASSERT(indexes.size() == nodes_.size()); nodes.clear(); nodes.reserve(indexes.size()); for (size_t idx : indexes) { nodes.push_back(nodes_[idx]); } } void RendezvousHash::hashOrder(const RendezvousHash::Key& key, std::vector& indices) { AutoLock lock(mutex_); hashOrderInternal(key, indices); } void RendezvousHash::hashOrderInternal(const RendezvousHash::Key& key, std::vector& indices) { if (nodes_.size() == 0) { throw BadParameter("Cannot return hashed order with no nodes", Here()); } // n.b. Does not do locking. That is delegated to the public calling functions. indices.resize(nodes_.size()); std::iota(indices.begin(), indices.end(), 0); // We hash based on a stringized key std::string skey = flatten(key); std::vector hashes; hashes.reserve(nodes_.size()); // Calculate hashes of the key with each of the available nodes for (const auto& node : nodes_) { std::string toHash = skey + "+" + node; hashes.emplace_back(hash_(toHash)); Log::debug() << "node=" << node << ", str=" << toHash << ", hash=" << hashes.back() << std::endl; } // TODO: We don't necessarily need N results, could do a subset. std::sort(indices.begin(), indices.end(), [&hashes](size_t lhs, size_t rhs) { return hashes[lhs] < hashes[rhs]; }); } bool RendezvousHash::addNode(const RendezvousHash::Node& node) { AutoLock lock(mutex_); auto it = std::find(nodes_.begin(), nodes_.end(), node); if (it == nodes_.end()) { nodes_.push_back(node); return true; } return false; } bool RendezvousHash::removeNode(const RendezvousHash::Node& node) { AutoLock lock(mutex_); auto it = std::find(nodes_.begin(), nodes_.end(), node); if (it != nodes_.end()) { nodes_.erase(it); return true; } return false; } std::string RendezvousHash::flatten(const RendezvousHash::Key& key) const { std::ostringstream flat; for (Key::const_iterator itr = key.begin(); itr != key.end(); ++itr) { flat << "/" << itr->first << ":" << itr->second; } return flat.str(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/Regex.cc0000664000175000017500000000750515161702250017420 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/utils/Regex.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Regex::Regex(const std::string& s, bool shell, bool extended) : str_(s), extended_(extended) { // Log::debug() << "Regex " << str_ << std::endl; if (shell) { long len = s.length() * 3 + 1; Buffer buffer(len); char* re = buffer; std::string::size_type i = 0; int j = 0; if (shell) { re[j++] = '^'; } while (i < s.length()) { switch (s[i]) { case '?': re[j++] = '.'; break; case '*': re[j++] = '.'; re[j++] = '*'; break; case '.': re[j++] = '\\'; re[j++] = '.'; break; case '[': re[j++] = '['; i++; while (i < s.length() && s[i] != ']') { re[j++] = s[i++]; } re[j++] = ']'; break; default: re[j++] = s[i]; break; } i++; ASSERT(j < len); } if (shell) { re[j++] = '$'; } re[j] = 0; str_ = re; } // Log::debug() << "Regex " << str_ << std::endl; compile(str_.c_str()); } Regex::~Regex() { regfree(&re_); } void Regex::print(std::ostream& s) const { s << "/" << str_ << "/"; } bool Regex::match(const std::string& s) const { regmatch_t pm; // Log::debug() << "Match " << s << " with " << str_ << " -> " << (regexec(&re_,s.c_str(),1,&pm,0) == 0) << // std::endl; return regexec(&re_, s.c_str(), 1, &pm, 0) == 0; } void Regex::compile(const char* p) { int n = regcomp(&re_, p, extended_ ? REG_EXTENDED : 0); if (n) { char buf[1024]; regerror(n, &re_, buf, sizeof(buf)); throw SeriousBug(buf); } } Regex::Regex(const Regex& other) : str_(other.str_), extended_(other.extended_) { compile(str_.c_str()); } Regex& Regex::operator=(const Regex& other) { regfree(&re_); str_ = other.str_; extended_ = other.extended_; compile(str_.c_str()); return *this; } std::string Regex::escape(std::string_view str) { std::string ret; // Reserve twice the size of str for worst-case ret.reserve(str.size() * 2); for (const char& c : str) { switch (c) { case '.': case '^': case '$': case '*': case '+': case '-': case '?': case '(': case ')': case '[': case ']': case '{': case '}': case '\\': case '|': ret.insert(ret.end(), '\\'); [[fallthrough]]; default: ret.insert(ret.end(), c); } } return ret; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/StringTools.cc0000664000175000017500000001523715161702250020636 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Tokenizer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- std::string StringTools::substitute(const std::string& s, const std::map& m) { std::string result; size_t len = s.length(); bool var = false; std::string word; std::map::const_iterator j; for (size_t i = 0; i < len; i++) { switch (s[i]) { case '{': if (var) { std::ostringstream os; os << "StringTools::substitute: unexpected { found in " << s << " at position " << i; throw UserError(os.str()); } var = true; word = ""; break; case '}': if (!var) { std::ostringstream os; os << "StringTools::substitute: unexpected } found in " << s << " at position " << i; throw UserError(os.str()); } var = false; j = m.find(word); if (j == m.end()) { std::ostringstream os; os << "StringTools::substitute: cannot find a value for '" << word << "' in " << s << " at position " << i; throw UserError(os.str()); } result += (*j).second; break; default: if (var) { word += s[i]; } else { result += s[i]; } break; } } if (var) { std::ostringstream os; os << "StringTools::substitute: missing } in " << s; throw UserError(os.str()); } return result; } std::vector StringTools::listVariables(const std::string& s) { std::vector result; size_t len = s.length(); bool var = false; std::string word; for (size_t i = 0; i < len; i++) { switch (s[i]) { case '{': if (var) { std::ostringstream os; os << "StringTools::listVariables: unexpected { found in " << s << " at position " << i; throw UserError(os.str()); } var = true; word = ""; break; case '}': if (!var) { std::ostringstream os; os << "StringTools::listVariables: unexpected } found in " << s << " at position " << i; throw UserError(os.str()); } var = false; result.push_back(word); break; default: if (var) { word += s[i]; } break; } } if (var) { std::ostringstream os; os << "StringTools::listVariables: missing } in " << s; throw UserError(os.str()); } return result; } std::string StringTools::upper(const std::string& v) { std::string r = v; std::transform(r.begin(), r.end(), r.begin(), static_cast(toupper)); return r; } std::string StringTools::lower(const std::string& v) { std::string r = v; std::transform(r.begin(), r.end(), r.begin(), static_cast(tolower)); return r; } std::string StringTools::trim(const std::string& str) { return trim(str, " \t\n"); } std::string StringTools::trim(const std::string& str, const std::string& chars) { size_t startpos = str.find_first_not_of(chars); size_t endpos = str.find_last_not_of(chars); if ((std::string::npos == startpos) || (std::string::npos == endpos)) { return ""; } return str.substr(startpos, endpos - startpos + 1); } std::string StringTools::front_trim(const std::string& str) { return front_trim(str, " \t"); } std::string StringTools::front_trim(const std::string& str, const std::string& chars) { size_t startpos = str.find_first_not_of(chars); if (std::string::npos == startpos) { return ""; } return str.substr(startpos); } std::string StringTools::back_trim(const std::string& str) { return back_trim(str, " \t"); } std::string StringTools::back_trim(const std::string& str, const std::string& chars) { size_t endpos = str.find_last_not_of(chars); if (std::string::npos == endpos) { return ""; } return str.substr(0, endpos + 1); } std::vector StringTools::split(const std::string& delim, const std::string& text) { std::vector ss; Tokenizer tokenizer(delim); tokenizer(text, ss); return ss; } bool StringTools::startsWith(const std::string& str, const std::string& substr) { if (substr.empty() || str.size() < substr.size()) { return false; } for (std::string::size_type i = 0; i < substr.size(); ++i) { if (substr[i] != str[i]) { return false; } } return true; } bool StringTools::beginsWith(const std::string& str, const std::string& substr) { return startsWith(str, substr); } bool StringTools::endsWith(const std::string& str, const std::string& substr) { if (substr.empty() || str.size() < substr.size()) { return false; } std::string::const_reverse_iterator rj = str.rbegin(); for (std::string::const_reverse_iterator ri = substr.rbegin(); ri != substr.rend(); ++ri, ++rj) { if (*ri != *rj) { return false; } } return true; } bool StringTools::isQuoted(const std::string& value) { return value.size() > 1 && ((value[0] == '"' && value[value.size() - 1] == '"') || (value[0] == '\'' && value[value.size() - 1] == '\'')); } std::string StringTools::unQuote(const std::string& value) { if (isQuoted(value)) { return value.substr(1, value.size() - 2); } return value; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/AECCompressor.cc0000664000175000017500000001325115161702250021006 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/utils/AECCompressor.h" #include extern "C" { // libaec.h has c linkage #include "libaec.h" } #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" // compression parameters heve been tuned for maximum compression ratio with GRIB2 3D fields. // Performance on GRIB 1 sfc fields is bad with any parameter values // https://sourceware.org/bzip2/manual/manual.html#bzcompress-init // // bits_per_sample: range [1..32] Storage size from sample size. If a sample requires less bits than the storage size // provides, then you have to make sure that unused bits are not set. Libaec does not check this for performance reasons // and will produce undefined output if unused bits are set. // 1 - 8 bits 1 byte // 9 - 16 bits 2 bytes // 17 - 24 bits 3 bytes (only if AEC_DATA_3BYTE is set) // 25 - 32 bits 4 bytes (if AEC_DATA_3BYTE is set) // 17 - 32 bits 4 bytes (if AEC_DATA_3BYTE is not set) // // block_size: range [8..64] Smaller blocks allow the compression to adapt more rapidly to changing source // statistics. Larger blocks create less overhead but can be less efficient if source statistics change across the // block. // // rsi: sets the reference sample interval. A large RSI will improve performance and efficiency. It will also increase // memory requirements since internal buffering is based on RSI size. A smaller RSI may be desirable in situations where // each RSI will be packetized and possible error propagation has to be minimized. #define AEC_bits_per_sample 16 #define AEC_block_size 64 #define AEC_rsi 129 #define AEC_flags AEC_DATA_PREPROCESS | AEC_DATA_MSB namespace eckit { //---------------------------------------------------------------------------------------------------------------------- inline void AECCall(int code, const char* aec_func, const eckit::CodeLocation& loc) { if (code != AEC_OK) { std::ostringstream msg; msg << "returned " << code; switch (code) { case AEC_CONF_ERROR: msg << " (AEC_CONF_ERROR)"; break; case AEC_STREAM_ERROR: msg << " (AEC_STREAM_ERROR)"; break; case AEC_DATA_ERROR: msg << " (AEC_DATA_ERROR)"; break; case AEC_MEM_ERROR: msg << " (AEC_MEM_ERROR)"; break; default: msg << " (UNRECOGNIZED ERROR)"; } throw FailedLibraryCall("AEC", aec_func, msg.str(), loc); } } #define AEC_CALL(a) AECCall(a, #a, Here()) //---------------------------------------------------------------------------------------------------------------------- AECCompressor::AECCompressor() {} AECCompressor::~AECCompressor() {} static size_t minInputSize(const size_t inputSize, const aec_stream& strm) { size_t blockSizeBytes = strm.bits_per_sample * strm.block_size / 8; size_t minSize = (inputSize / blockSizeBytes); if (inputSize % blockSizeBytes > 0) { minSize++; } return minSize * blockSizeBytes; } size_t AECCompressor::compress(const void* inTmp, size_t len, Buffer& out) const { struct aec_stream strm; strm.bits_per_sample = AEC_bits_per_sample; strm.block_size = AEC_block_size; strm.rsi = AEC_rsi; strm.flags = AEC_flags; Buffer in(minInputSize(len, strm)); in.copy(inTmp, len); if (in.size() > len) { ::memset(in + len, 0, in.size() - len); } size_t maxcompressed = size_t(1.2 * in.size()); if (out.size() < maxcompressed) { out.resize(maxcompressed); } strm.next_in = (unsigned char*)in.data(); strm.avail_in = in.size(); strm.next_out = (unsigned char*)out.data(); strm.avail_out = out.size(); AEC_CALL(aec_encode_init(&strm)); // Perform encoding in one call and flush output. // you must be sure that the output buffer is large enough for all compressed output AEC_CALL(aec_encode(&strm, AEC_FLUSH)); size_t outSize = strm.total_out; // free all resources used by encoder AEC_CALL(aec_encode_end(&strm)); return outSize; } void AECCompressor::uncompress(const void* in, size_t len, Buffer& out, size_t outlen) const { struct aec_stream strm; strm.bits_per_sample = AEC_bits_per_sample; strm.block_size = AEC_block_size; strm.rsi = AEC_rsi; strm.flags = AEC_flags; strm.next_in = (const unsigned char*)(in); strm.avail_in = len; size_t outSize = minInputSize(outlen, strm); // If out is sized large enough, we can use its extra capacity. // Otherwise we allocate a temporary outTmp and at end we move outTmp into out. Buffer outTmp; if (out.size() >= outSize) { strm.next_out = (unsigned char*)out.data(); } else { outTmp.resize(outSize); strm.next_out = (unsigned char*)outTmp.data(); } strm.avail_out = outSize; AEC_CALL(aec_decode_init(&strm)); AEC_CALL(aec_decode(&strm, AEC_FLUSH)); ASSERT(strm.total_out == outSize); AEC_CALL(aec_decode_end(&strm)); if (outTmp.size()) { out = std::move(outTmp); } } CompressorBuilder aec("aec"); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/Regex.h0000664000175000017500000000305215161702250017253 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Regex.h // Baudouin Raoult - ECMWF Jan 98 #ifndef eckit_Regex_h #define eckit_Regex_h #include #include #include namespace eckit { //-------------------------------------------------------------------------------------------------- class Regex { public: // -- Contructors Regex(const std::string& = ".*", bool shell = false, bool extended = true); Regex(const Regex&); ~Regex(); // -- Methods Regex& operator=(const Regex&); bool match(const std::string& s) const; operator const std::string&() const { return str_; } bool operator==(const Regex& other) const { return str_ == other.str_; } static std::string escape(std::string_view); protected: // methods void print(std::ostream&) const; private: // members std::string str_; regex_t re_; bool extended_; private: // methods void compile(const char*); friend std::ostream& operator<<(std::ostream& s, const Regex& p) { p.print(s); return s; } }; //-------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/utils/Clock.h0000664000175000017500000000117415161702250017237 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_Clock_h #define eckit_utils_Clock_h #include "time.h" namespace eckit { class Clock { public: /// @returns UNIX Time since Epoch static time_t now() { return ::time(nullptr); } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/utils/Tokenizer.h0000664000175000017500000000572615161702250020165 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Manuel Fuentes /// @author Tiago Quintino #ifndef eckit_Tokenizer_h #define eckit_Tokenizer_h #include #include #include namespace eckit { //--------------------------------------------------------------------------------------------------------------------- class Tokenizer { public: // methods Tokenizer(char, bool keepEmpty = false); Tokenizer(const std::string&, bool keepEmpty = false); Tokenizer(const Tokenizer&) = delete; Tokenizer& operator=(const Tokenizer&) = delete; Tokenizer(Tokenizer&&) = delete; Tokenizer& operator=(Tokenizer&&) = delete; ~Tokenizer(); void operator()(const std::string&, std::vector&) const; void operator()(std::istream&, std::vector&) const; void operator()(const std::string&, std::set&) const; void operator()(std::istream&, std::set&) const; std::vector tokenize(const std::string&) const; std::vector tokenize(std::istream&) const; /** * Splits the given the string on the first instance of the separator. * * The result is the set of separated tokens: * - if the separator is not found, containing 1 token: [begin, end] * - if the separator is found, containing 2 tokens: [begin, separator[, ]separator, end] * */ static std::vector split_at(const std::string& s, char separator); private: std::set > separator_; // To make searching faster bool keepEmpty_; private: void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const Tokenizer& p) { p.print(s); return s; } }; //--------------------------------------------------------------------------------------------------------------------- inline std::vector Tokenizer::tokenize(const std::string& s) const { std::vector r; this->operator()(s, r); return r; } inline std::vector Tokenizer::tokenize(std::istream& s) const { std::vector r; this->operator()(s, r); return r; } inline std::vector Tokenizer::split_at(const std::string& s, char separator) { if (auto found = s.find_first_of(separator); found != std::string::npos) { return {s.substr(0, found), s.substr(found + 1)}; } return {s}; } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/utils/SnappyCompressor.cc0000664000175000017500000000465015161702250021673 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/utils/SnappyCompressor.h" // #include "snappy.h" #include "snappy-c.h" // header includes extern c linkage #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- SnappyCompressor::SnappyCompressor() {} SnappyCompressor::~SnappyCompressor() {} size_t SnappyCompressor::compress(const void* in, size_t len, Buffer& out) const { size_t maxcompressed = snappy_max_compressed_length(len); if (out.size() < maxcompressed) { out.resize(maxcompressed); } size_t outlen = out.size(); snappy_status status = snappy_compress(static_cast(in), len, out, &outlen); if (status == SNAPPY_OK) { return outlen; } std::ostringstream msg; if (status == SNAPPY_INVALID_INPUT) { msg << "invalid input to compress"; } if (status == SNAPPY_BUFFER_TOO_SMALL) { msg << "output buffer too small, size " << out.size(); } throw FailedLibraryCall("snappy", "compress", msg.str(), Here()); } void SnappyCompressor::uncompress(const void* in, size_t len, Buffer& out, size_t outlen) const { snappy_status status; if (out.size() < outlen) { out.resize(outlen); } size_t uncompressed = out.size(); status = snappy_uncompress(static_cast(in), len, out, &uncompressed); ASSERT(uncompressed == outlen); if (status != SNAPPY_OK) { std::ostringstream msg; if (status == SNAPPY_INVALID_INPUT) { msg << "invalid input to compress"; } if (status == SNAPPY_BUFFER_TOO_SMALL) { msg << "output buffer too small, size " << out.size(); } throw FailedLibraryCall("snappy", "compress", msg.str(), Here()); } } CompressorBuilder snappy("snappy"); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/Translator.cc0000664000175000017500000002147315161702250020477 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static unsigned long long multiplier(const char* p) { while (isspace(*p)) { p++; } if (*p && *(p + 1)) { // cater for GB or GiB (SI units) if (towlower(*(p + 1)) == 'b' || (towlower(*(p + 1)) == 'i' && towlower(*(p + 2)) == 'b')) { switch (towlower(*p)) { case 'k': return (1LL << 10); case 'm': return (1LL << 20); case 'g': return (1LL << 30); case 't': return (1LL << 40); case 'p': return (1LL << 50); case 'e': return (1LL << 60); // case 'z': return (1LL << 70); // case 'y': return (1LL << 80); } } } return 1; } std::string Translator::operator()(bool value) { std::ostringstream s; s << value; return s.str(); } bool Translator::operator()(const std::string& str) { std::string s = StringTools::lower(str); if (s == "no" || s == "off" || s == "false" || s == "0") { return false; } if (s == "yes" || s == "on" || s == "true" || s == "1") { return true; } // Catter for ints return atoi(s.c_str()); // 0 is returned on non-conversion } std::string Translator::operator()(int value) { std::ostringstream s; s << value; return s.str(); } std::string Translator::operator()(unsigned int value) { std::ostringstream s; s << value; return s.str(); } int Translator::operator()(const std::string& s) { if (s == "no" || s == "off" || s == "false") { return false; } if (s == "yes" || s == "on" || s == "true") { return true; } // Catter for ints char* more; int result = strtol(s.c_str(), &more, 10); return result * multiplier(more); } unsigned int Translator::operator()(const std::string& s) { if (s == "no" || s == "off" || s == "false") { return false; } if (s == "yes" || s == "on" || s == "true") { return true; } // Catter for ints char* more; unsigned int result = strtoul(s.c_str(), &more, 10); return result * multiplier(more); } std::string Translator::operator()(long value) { std::ostringstream s; s << value; return s.str(); } long Translator::operator()(const std::string& s) { char* more; long result = strtol(s.c_str(), &more, 10); return result * multiplier(more); } short Translator::operator()(const std::string& s) { char* more; long result = strtol(s.c_str(), &more, 10); result = result * multiplier(more); ASSERT(short(result) == result); return result; } unsigned char Translator::operator()(const std::string& s) { char* more; long result = strtol(s.c_str(), &more, 10); result = result * multiplier(more); ASSERT(static_cast(result) == result); return result; } std::string Translator::operator()(unsigned char value) { std::ostringstream s; s << value; return s.str(); } std::string Translator::operator()(short value) { std::ostringstream s; s << value; return s.str(); } std::string Translator::operator()(float value) { std::ostringstream s; s << value; return s.str(); } std::string Translator::operator()(double value) { std::ostringstream s; s << value; return s.str(); } double Translator::operator()(const std::string& s) { char* pend; errno = 0; double d = ::strtod(s.c_str(), &pend); // ECKIT_DEBUG_VAR( d ); // ECKIT_DEBUG_VAR( s ); // ECKIT_DEBUG_VAR( s.size() ); // ECKIT_DEBUG_VAR( pend - s.c_str() ); // ECKIT_DEBUG_VAR( errno ); if (s.empty() || s[0] == ' ' || static_cast(pend - s.c_str()) != s.size() || (errno != 0)) { throw BadParameter("Bad conversion from std::string '" + s + "' to double", Here()); } return d; // return atof(s.c_str()); } float Translator::operator()(const std::string& s) { char* pend; errno = 0; float f = ::strtof(s.c_str(), &pend); // ECKIT_DEBUG_VAR( d ); // ECKIT_DEBUG_VAR( s ); // ECKIT_DEBUG_VAR( s.size() ); // ECKIT_DEBUG_VAR( pend - s.c_str() ); // ECKIT_DEBUG_VAR( errno ); if (s.empty() || s[0] == ' ' || static_cast(pend - s.c_str()) != s.size() || (errno != 0)) { throw BadParameter("Bad conversion from std::string '" + s + "' to float", Here()); } return f; // return atof(s.c_str()); } unsigned long Translator::operator()(const std::string& s) { char* more; unsigned long result = strtoul(s.c_str(), &more, 10); return result * multiplier(more); } std::string Translator::operator()(unsigned long value) { std::ostringstream s; s << value; return s.str(); } unsigned long long Translator::operator()(const std::string& s) { char* more; unsigned long long result = strtoull(s.c_str(), &more, 10); return result * multiplier(more); } std::string Translator::operator()(unsigned long long value) { std::ostringstream s; s << value; return s.str(); } long long Translator::operator()(const std::string& s) { char* more; long long result = strtoll(s.c_str(), &more, 10); return result * multiplier(more); } std::string Translator::operator()(long long value) { std::ostringstream s; s << value; return s.str(); } std::vector Translator >::operator()(const std::string& s) { std::vector result; Tokenizer parse(", \t"); parse(s, result); return result; } std::vector Translator >::operator()(const std::string& s) { std::vector r; Tokenizer parse(", \t"); parse(s, r); std::vector result; for (size_t i = 0; i < r.size(); i++) { result.push_back(Translator()(r[i])); } return result; } std::string Translator, std::string>::operator()(const std::vector& v) { std::string result; for (size_t i = 0; i < v.size(); ++i) { if (i) { result += " "; } result += Translator()(v[i]); } return result; } std::string Translator, std::string>::operator()(const std::vector& v) { std::string result; for (size_t i = 0; i < v.size(); ++i) { if (i) { result += ","; } result += v[i]; } return result; } std::set Translator >::operator()(const std::string& s) { std::vector v; Tokenizer parse(", \t"); parse(s, v); std::set result(v.begin(), v.end()); return result; } std::string Translator, std::string>::operator()(const std::set& v) { std::string result; for (std::set::const_iterator i = v.begin(); i != v.end(); ++i) { if (result.length()) { result += ","; } result += *i; } return result; } char Translator::operator()(const std::string& s) { ASSERT(s.length() == 1); return s[0]; } std::string Translator::operator()(char c) { std::string s; s = c; return s; } std::string Translator::operator()(signed char v) { return Translator{}(static_cast(v)); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/MD4.h0000664000175000017500000000243215161702250016566 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_MD4_H #define eckit_utils_MD4_H #include "eckit/eckit.h" #if eckit_HAVE_SSL #include #else #error "eckit was not configured with OpenSSL, MD4 is disabled. Use conditional eckit_HAVE_SSL from eckit/eckit.h" #endif #ifndef MD4_DIGEST_LENGTH #define MD4_DIGEST_LENGTH 16 #endif #include "eckit/utils/Hash.h" namespace eckit { class MD4 : public Hash { public: // types MD4(); explicit MD4(const char*); explicit MD4(const std::string&); MD4(const void* data, size_t len); ~MD4() override; void reset() const override; digest_t compute(const void*, long) override; void update(const void*, long) override; digest_t digest() const override; template MD4& operator<<(const T& x) { add(x); return *this; } private: // members mutable MD4_CTX ctx_; }; } // end namespace eckit #endif eckit-2.0.7/src/eckit/utils/HyperCube.h0000664000175000017500000000627415161702250020100 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Oct 96 #ifndef eckit_utils_HyperCube_h #define eckit_utils_HyperCube_h #include #include "eckit/exception/Exceptions.h" #include "eckit/types/Types.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// Helper class to handle multi-dimension objects /// @note The first dimension should be the one most likely to change class HyperCube { public: // types using Dimensions = std::vector; using Coordinates = std::vector; using Remapping = std::vector; public: // methods explicit HyperCube(const Dimensions& d) : dimensions_(d) {} /// Translate coordinates into an index to a 1 dimension array Ordinal index(const Coordinates&) const; /// @returns the number of elemets Ordinal count() const; /// Translate index to coordinates void coordinates(Ordinal index, Coordinates&) const; // Accessors const Dimensions& dimensions() const { return dimensions_; } Dimensions& dimensions() { return dimensions_; } Ordinal dimensions(Ordinal n) const { return dimensions_[n]; } Ordinal size() const { return dimensions_.size(); } /// @returns the 'remapping' std::vector needing to add 'count' labels for the dimension 'which' at position 'where' HyperCube addToDimension(Ordinal which, Ordinal where, Ordinal count, Remapping&) const; /// Combines two 'remapping' vectors static void combine(Remapping&, const Remapping&); private: // members Dimensions dimensions_; }; //---------------------------------------------------------------------------------------------------------------------- /// Method is inlined for speed inline Ordinal HyperCube::count() const { return std::accumulate(dimensions_.begin(), dimensions_.end(), 1, std::multiplies()); } /// Method is inlined for speed inline Ordinal HyperCube::index(const Coordinates& coord) const { Ordinal n = 1; Ordinal a = 0; ASSERT(coord.size() == dimensions_.size()); // The fact that this is in reverse is important for addToDimension for (int i = coord.size() - 1; i >= 0; i--) { ASSERT(/*coord[i] >= 0 &&*/ coord[i] < dimensions_[i]); a += coord[i] * n; n *= dimensions_[i]; } return a; } /// Method is inlined for speed inline void HyperCube::combine(Remapping& map1, const Remapping& map2) { if (map1.size() == 0) { map1 = map2; } else { for (Remapping::iterator i = map1.begin(); i != map1.end(); ++i) { ASSERT(*i < map2.size()); *i = map2[*i]; } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/utils/BZip2Compressor.cc0000664000175000017500000001110415161702250021337 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/utils/BZip2Compressor.h" #include #include "bzlib.h" // header includes extern c linkage #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- inline void BZip2Call(int code, const char* bzip2_func, const eckit::CodeLocation& loc) { if (code < 0) { std::ostringstream msg; msg << "returned " << code; switch (code) { case BZ_SEQUENCE_ERROR: msg << " (BZ_SEQUENCE_ERROR)"; break; case BZ_PARAM_ERROR: msg << " (BZ_PARAM_ERROR)"; break; case BZ_MEM_ERROR: msg << " (BZ_MEM_ERROR)"; break; case BZ_DATA_ERROR: msg << " (BZ_DATA_ERROR)"; break; case BZ_DATA_ERROR_MAGIC: msg << " (BZ_DATA_ERROR_MAGIC)"; break; case BZ_IO_ERROR: msg << " (BZ_IO_ERROR)"; break; case BZ_UNEXPECTED_EOF: msg << " (BZ_UNEXPECTED_EOF)"; break; case BZ_OUTBUFF_FULL: msg << " (BZ_OUTBUFF_FULL)"; break; case BZ_CONFIG_ERROR: msg << " (BZ_CONFIG_ERROR)"; break; default: msg << " (UNRECOGNIZED ERROR)"; } throw FailedLibraryCall("BZlib2", bzip2_func, msg.str(), loc); } } #define BZ2_CALL(a) BZip2Call(a, #a, Here()) //---------------------------------------------------------------------------------------------------------------------- BZip2Compressor::BZip2Compressor() {} BZip2Compressor::~BZip2Compressor() {} size_t BZip2Compressor::compress(const void* in, size_t len, Buffer& out) const { std::ostringstream msg; // https://sourceware.org/bzip2/manual/manual.html#bzbufftobuffcompress // To guarantee that the compressed data will fit in its buffer, allocate an output buffer of size 1% larger than // the uncompressed data, plus six hundred extra bytes. size_t maxcompressed = (size_t)(1.01 * len + 600); if (out.size() < maxcompressed) { out.resize(maxcompressed); } size_t bufferSize = out.size(); ASSERT(len < std::numeric_limits::max()); ASSERT(maxcompressed < std::numeric_limits::max()); ASSERT(bufferSize < std::numeric_limits::max()); bz_stream strm; strm.avail_in = 0UL; strm.next_in = nullptr; strm.next_out = nullptr; strm.bzalloc = nullptr; strm.bzfree = nullptr; strm.opaque = nullptr; /// @note see https://sourceware.org/bzip2/manual/manual.tml#bzcompress-init BZ2_CALL(BZ2_bzCompressInit(&strm, 9, 0, 30)); strm.next_in = (char*)in; strm.avail_in = len; strm.next_out = (char*)out.data(); strm.avail_out = bufferSize; BZ2_CALL(BZ2_bzCompress(&strm, BZ_FINISH)); size_t outSize = bufferSize - strm.avail_out; strm.avail_in = 0; strm.next_in = nullptr; BZ2_CALL(BZ2_bzCompressEnd(&strm)); return outSize; } void BZip2Compressor::uncompress(const void* in, size_t len, Buffer& out, size_t outlen) const { ASSERT(len < std::numeric_limits::max()); if (out.size() < outlen) { out.resize(outlen); } bz_stream strm; strm.avail_in = 0UL; strm.next_in = nullptr; strm.next_out = nullptr; strm.bzalloc = nullptr; strm.bzfree = nullptr; strm.opaque = nullptr; BZ2_CALL(BZ2_bzDecompressInit(&strm, 0, 0)); strm.next_in = (char*)in; strm.avail_in = len; strm.next_out = (char*)out.data(); strm.avail_out = outlen; unsigned int bufferSize = outlen; BZ2_CALL(BZ2_bzDecompress(&strm)); size_t outSize = bufferSize - strm.avail_out; ASSERT(outSize == outlen); strm.next_out = nullptr; BZ2_CALL(BZ2_bzDecompressEnd(&strm)); } CompressorBuilder bzip2("bzip2"); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/utils/SHA1.h0000664000175000017500000000244415161702250016701 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_utils_SHA1_H #define eckit_utils_SHA1_H #include "eckit/eckit.h" #if eckit_HAVE_SSL #include #else #error "eckit was not configured with OpenSSL, SHA1 is disabled. Use conditional eckit_HAVE_SSL from eckit/eckit.h" #endif #ifndef SHA_DIGEST_LENGTH #define SHA_DIGEST_LENGTH 20 #endif #include "eckit/utils/Hash.h" namespace eckit { class SHA1 : public Hash { public: // types SHA1(); explicit SHA1(const char*); explicit SHA1(const std::string&); SHA1(const void* data, size_t len); ~SHA1() override; void reset() const override; digest_t compute(const void*, long) override; void update(const void*, long) override; digest_t digest() const override; template SHA1& operator<<(const T& x) { add(x); return *this; } private: // members mutable SHA_CTX ctx_; }; } // end namespace eckit #endif eckit-2.0.7/src/eckit/value/0000775000175000017500000000000015161702250016004 5ustar alastairalastaireckit-2.0.7/src/eckit/value/Content.cc0000664000175000017500000003205115161702250017726 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/Content.h" #include "eckit/exception/Exceptions.h" #include "eckit/os/BackTrace.h" #include "eckit/value/Value.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { class BadConversion : public Exception { public: BadConversion(const std::string& w, const CodeLocation& loc); }; class BadComparison : public Exception { public: BadComparison(const std::string& w, const CodeLocation& loc); }; class BadOperator : public Exception { public: BadOperator(const std::string& w, const CodeLocation& loc); }; BadConversion::BadConversion(const std::string& w, const CodeLocation& loc) : Exception(std::string("Bad Conversion: ") + w, loc) { // std::cout << what() << std::endl; // std::cout << BackTrace::dump() << std::endl; } BadComparison::BadComparison(const std::string& w, const CodeLocation& loc) : Exception(std::string("Bad Comparison: ") + w, loc) { // std::cout << what() << std::endl; // std::cout << BackTrace::dump() << std::endl; } BadOperator::BadOperator(const std::string& w, const CodeLocation& loc) : Exception(std::string("Bad operator: ") + w, loc) { // std::cout << what() << std::endl; // std::cout << BackTrace::dump() << std::endl; } //---------------------------------------------------------------------------------------------------------------------- ClassSpec Content::classSpec_ = { &Streamable::classSpec(), "Content", }; Reanimator Content::reanimator_; void Content::encode(Stream& s) const { Streamable::encode(s); } Content::Content(Stream& s) : Streamable(s) {} Content::~Content() {} void Content::badConversion(const std::string& to) const { std::ostringstream s; s << "Cannot convert " << *this << " (" << typeName() << ") to " << to; throw BadConversion(s.str(), Here()); } void Content::badComparison(const std::string& to) const { std::ostringstream s; s << "Cannot compare " << *this << " (" << typeName() << ") with " << to; throw BadComparison(s.str(), Here()); } void Content::badOperator(const std::string& op, const std::string& to) const { std::ostringstream s; s << *this << " (" << typeName() << ") " << op << " " << to; throw BadOperator(s.str(), Here()); } Value Content::remove(const Value&) { std::ostringstream s; s << *this << " (" << typeName() << ") method 'remove' not implemented"; throw BadOperator(s.str(), Here()); } void Content::append(const Value&) { std::ostringstream s; s << *this << " (" << typeName() << ") method 'append' not implemented"; throw BadOperator(s.str(), Here()); } Value& Content::element(const Value&) { std::ostringstream s; s << *this << " (" << typeName() << ") method 'element' not implemented"; throw BadOperator(s.str(), Here()); } Value Content::keys() const { std::ostringstream s; s << *this << " (" << typeName() << ") method 'keys' not implemented"; throw BadOperator(s.str(), Here()); } size_t Content::size() const { std::ostringstream s; s << *this << " (" << typeName() << ") method 'size' not implemented"; throw BadOperator(s.str(), Here()); } bool Content::contains(const Value&) const { std::ostringstream s; s << *this << " (" << typeName() << ") method 'contains' not implemented"; throw BadOperator(s.str(), Here()); } Value Content::negate() const { std::ostringstream s; s << *this << " (" << typeName() << ") method 'negate' not implemented"; throw BadOperator(s.str(), Here()); } void Content::value(long long&) const { badConversion("long long"); } void Content::value(bool&) const { badConversion("bool"); } void Content::value(double&) const { badConversion("double"); } void Content::value(std::string&) const { badConversion("std::string"); } void Content::value(Date&) const { badConversion("Date"); } void Content::value(Time&) const { badConversion("Time"); } void Content::value(DateTime&) const { badConversion("DateTime"); } void Content::value(ValueMap&) const { badConversion("Map"); } void Content::value(ValueList& v) const { // Cast away constness, so the Contnt can be attached by the value v.push_back(Value(const_cast(this))); } bool Content::operator==(const Content& other) const { return (this->compare(other) == 0); } bool Content::operator<(const Content& other) const { return (this->compare(other) < 0); } int Content::compareNumber(const NumberContent&) const { badComparison("Number"); return 0; } int Content::compareBool(const BoolContent&) const { badComparison("Bool"); return 0; } int Content::compareDouble(const DoubleContent&) const { badComparison("Double"); return 0; } int Content::compareString(const StringContent&) const { badComparison("String"); return 0; } int Content::compareNil(const NilContent&) const { badComparison("Nil"); return 0; } int Content::compareList(const ListContent&) const { badComparison("List"); return 0; } int Content::compareMap(const MapContent&) const { badComparison("Map"); return 0; } int Content::compareDate(const DateContent&) const { badComparison("Date"); return 0; } int Content::compareTime(const TimeContent&) const { badComparison("Time"); return 0; } int Content::compareDateTime(const DateTimeContent&) const { badComparison("DateTime"); return 0; } int Content::compareOrderedMap(const OrderedMapContent&) const { badComparison("OrderedMap"); return 0; } Content* Content::operator+(const Content& other) const { return add(other); } Content* Content::add(const Content& other) const { badOperator("+", other.typeName()); return nullptr; } Content* Content::addNumber(const NumberContent&) const { badOperator("+", "Number"); return nullptr; } Content* Content::addBool(const BoolContent&) const { badOperator("+", "Bool"); return nullptr; } Content* Content::addDouble(const DoubleContent&) const { badOperator("+", "Number"); return nullptr; } Content* Content::addString(const StringContent&) const { badOperator("+", "String"); return nullptr; } Content* Content::addNil(const NilContent&) const { badOperator("+", "Nil"); return nullptr; } Content* Content::addList(const ListContent&) const { badOperator("+", "List"); return nullptr; } Content* Content::addMap(const MapContent&) const { badOperator("+", "List"); return nullptr; } Content* Content::addDate(const DateContent&) const { badOperator("+", "Date"); return nullptr; } Content* Content::addTime(const TimeContent&) const { badOperator("+", "Time"); return nullptr; } Content* Content::addDateTime(const DateTimeContent&) const { badOperator("+", "DateTime"); return nullptr; } Content* Content::addOrderedMap(const OrderedMapContent&) const { badOperator("+", "OrderedMap"); return nullptr; } Content* Content::operator-(const Content& other) const { return sub(other); } Content* Content::sub(const Content& other) const { badOperator("-", other.typeName()); return nullptr; } Content* Content::subNumber(const NumberContent&) const { badOperator("-", "Number"); return nullptr; } Content* Content::subDouble(const DoubleContent&) const { badOperator("-", "Double"); return nullptr; } Content* Content::subBool(const BoolContent&) const { badOperator("-", "Bool"); return nullptr; } Content* Content::subString(const StringContent&) const { badOperator("-", "String"); return nullptr; } Content* Content::subNil(const NilContent&) const { badOperator("-", "Nil"); return nullptr; } Content* Content::subList(const ListContent&) const { badOperator("-", "List"); return nullptr; } Content* Content::subMap(const MapContent&) const { badOperator("-", "Map"); return nullptr; } Content* Content::subDate(const DateContent&) const { badOperator("-", "Date"); return nullptr; } Content* Content::subTime(const TimeContent&) const { badOperator("-", "Time"); return nullptr; } Content* Content::subDateTime(const DateTimeContent&) const { badOperator("-", "DateTime"); return nullptr; } Content* Content::subOrderedMap(const OrderedMapContent&) const { badOperator("-", "OrderedMap"); return nullptr; } Content* Content::operator*(const Content& other) const { return mul(other); } Content* Content::mul(const Content& other) const { badOperator("*", other.typeName()); return nullptr; } Content* Content::mulNumber(const NumberContent&) const { badOperator("*", "Number"); return nullptr; } Content* Content::mulBool(const BoolContent&) const { badOperator("*", "Bool"); return nullptr; } Content* Content::mulDouble(const DoubleContent&) const { badOperator("*", "Double"); return nullptr; } Content* Content::mulString(const StringContent&) const { badOperator("*", "String"); return nullptr; } Content* Content::mulNil(const NilContent&) const { badOperator("*", "Nil"); return nullptr; } Content* Content::mulList(const ListContent&) const { badOperator("*", "List"); return nullptr; } Content* Content::mulMap(const MapContent&) const { badOperator("*", "Map"); return nullptr; } Content* Content::mulDate(const DateContent&) const { badOperator("*", "Date"); return nullptr; } Content* Content::mulTime(const TimeContent&) const { badOperator("*", "Time"); return nullptr; } Content* Content::mulDateTime(const DateTimeContent&) const { badOperator("*", "DateTime"); return nullptr; } Content* Content::mulOrderedMap(const OrderedMapContent&) const { badOperator("-", "OrderedMap"); return nullptr; } Content* Content::operator/(const Content& other) const { return div(other); } Content* Content::div(const Content& other) const { badOperator("/", other.typeName()); return nullptr; } Content* Content::divDouble(const DoubleContent&) const { badOperator("/", "Double"); return nullptr; } Content* Content::divNumber(const NumberContent&) const { badOperator("/", "Number"); return nullptr; } Content* Content::divBool(const BoolContent&) const { badOperator("/", "Bool"); return nullptr; } Content* Content::divString(const StringContent&) const { badOperator("/", "String"); return nullptr; } Content* Content::divNil(const NilContent&) const { badOperator("/", "Nil"); return nullptr; } Content* Content::divList(const ListContent&) const { badOperator("/", "List"); return nullptr; } Content* Content::divMap(const MapContent&) const { badOperator("/", "Map"); return nullptr; } Content* Content::divDate(const DateContent&) const { badOperator("/", "Date"); return nullptr; } Content* Content::divTime(const TimeContent&) const { badOperator("/", "Time"); return nullptr; } Content* Content::divDateTime(const DateTimeContent&) const { badOperator("/", "DateTime"); return nullptr; } Content* Content::divOrderedMap(const OrderedMapContent&) const { badOperator("/", "OrderedMap"); return nullptr; } Content* Content::mod(const Content& other) const { badOperator("%", other.typeName()); return nullptr; } Content* Content::modDouble(const DoubleContent&) const { badOperator("%", "Double"); return nullptr; } Content* Content::modNumber(const NumberContent&) const { badOperator("%", "Number"); return nullptr; } Content* Content::modBool(const BoolContent&) const { badOperator("%", "Bool"); return nullptr; } Content* Content::modString(const StringContent&) const { badOperator("%", "String"); return nullptr; } Content* Content::modNil(const NilContent&) const { badOperator("%", "Nil"); return nullptr; } Content* Content::modList(const ListContent&) const { badOperator("%", "List"); return nullptr; } Content* Content::modMap(const MapContent&) const { badOperator("%", "Map"); return nullptr; } Content* Content::modDate(const DateContent&) const { badOperator("%", "Date"); return nullptr; } Content* Content::modTime(const TimeContent&) const { badOperator("%", "Time"); return nullptr; } Content* Content::modDateTime(const DateTimeContent&) const { badOperator("%", "DateTime"); return nullptr; } Content* Content::modOrderedMap(const OrderedMapContent&) const { badOperator("%", "OrderedMap"); return nullptr; } Content::Content() {} #ifndef IBM template <> Streamable* Reanimator::ressucitate(Stream&) const { return nullptr; } #endif //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/CompositeParams.cc0000664000175000017500000000452615161702250021430 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/CompositeParams.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- CompositeParams::CompositeParams() : plist_() {} CompositeParams::CompositeParams(const Params::List& plist) : plist_(plist) {} CompositeParams::CompositeParams(Stream& s) { Params::List::size_type len; s >> len; for (Params::List::size_type i = 0; i < len; ++i) { push_back(Params::decode(s)); } } CompositeParams& CompositeParams::push_front(const Params& p) { plist_.push_front(p); return *this; } CompositeParams& CompositeParams::push_back(const Params& p) { plist_.push_back(p); return *this; } Params::value_t getValue(const CompositeParams& p, const Params::key_t& key) { for (Params::List::const_iterator citr = p.plist_.begin(); citr != p.plist_.end(); ++citr) { Value v = getValue(*citr, key); if (!v.isNil()) { return v; } } return Value(); } //---------------------------------------------------------------------------------------------------------------------- void print(const CompositeParams& p, std::ostream& s) { for (Params::List::const_iterator citr = p.plist_.begin(); citr != p.plist_.end(); ++citr) { print(*citr, s); } } void encode(const CompositeParams& p, Stream& s) { s << p.plist_.size(); for (Params::List::const_iterator citr = p.plist_.begin(); citr != p.plist_.end(); ++citr) { encode(*citr, s); } } //---------------------------------------------------------------------------------------------------------------------- Params::Factory compositeParamsFactory; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/DoubleContent.cc0000664000175000017500000000756715161702250021077 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/JSON.h" #include "eckit/maths/Functions.h" #include "eckit/utils/Translator.h" #include "eckit/value/DoubleContent.h" #include "eckit/value/NumberContent.h" #include "eckit/utils/Hash.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec DoubleContent::classSpec_ = { &Content::classSpec(), "DoubleContent", }; Reanimator DoubleContent::reanimator_; DoubleContent::DoubleContent(double l) : value_(l) {} DoubleContent::DoubleContent(Stream& s) : Content(s), value_(0) { s >> value_; } Content* DoubleContent::clone() const { return new DoubleContent(value_); } void DoubleContent::encode(Stream& s) const { Content::encode(s); s << value_; } DoubleContent::~DoubleContent() {} void DoubleContent::print(std::ostream& s) const { s << value_; } void DoubleContent::json(JSON& s) const { s << value_; } int DoubleContent::compare(const Content& other) const { return -other.compareDouble(*this); } int DoubleContent::compareDouble(const DoubleContent& other) const { double diff = (value_ - other.value_); return eckit::sign(diff); } int DoubleContent::compareNumber(const NumberContent& other) const { double diff = (value_ - other.value_); return eckit::sign(diff); } void DoubleContent::value(double& l) const { l = value_; } void DoubleContent::value(long long& l) const { // Avoid FE_INVALID by checking if value_ is within valid range of long long constexpr double min = double(std::numeric_limits::min()); constexpr double max = double(std::numeric_limits::max()); if (value_ <= min || value_ >= max) { Content::value(l); // throws BadConversion return; } // Now safe to assign without FE_INVALID l = value_; if (l != value_) { Content::value(l); // throws BadConversion } } void DoubleContent::value(std::string& s) const { s = Translator()(value_); } Content* DoubleContent::add(const Content& other) const { return other.addDouble(*this); } Content* DoubleContent::addDouble(const DoubleContent& other) const { return new DoubleContent(other.value_ + value_); } Content* DoubleContent::sub(const Content& other) const { return other.subDouble(*this); } Content* DoubleContent::subDouble(const DoubleContent& other) const { return new DoubleContent(other.value_ - value_); } Content* DoubleContent::mul(const Content& other) const { return other.mulDouble(*this); } Content* DoubleContent::mulDouble(const DoubleContent& other) const { return new DoubleContent(other.value_ * value_); } Content* DoubleContent::div(const Content& other) const { return other.divDouble(*this); } Content* DoubleContent::mod(const Content& other) const { return other.modDouble(*this); } Content* DoubleContent::divDouble(const DoubleContent& other) const { return new DoubleContent(other.value_ / value_); } Value DoubleContent::negate() const { return Value(-value_); } void DoubleContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { while (depth-- > 0) { out << ' '; } } out << "double(" << value_ << ")"; } void DoubleContent::hash(Hash& h) const { h.add(value_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/Params.cc0000664000175000017500000000610115161702250017534 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/Params.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- bool Params::has(const std::string& key) const { return !getValue(*this, key).isNil(); } /* bool Params::get(const std::string& name, std::string& value) const { if(!has(name)) return false; value = std::string(operator[](name)); return true; } bool Params::get(const std::string& name, bool& value) const { if(!has(name)) return false; value = operator[](name); return true; } bool Params::get(const std::string& name, long& value) const { if(!has(name)) return false; value = operator[](name); return true; } bool Params::get(const std::string& name, size_t& value) const { if(!has(name)) return false; value = operator[](name); return true; } bool Params::get(const std::string& name, double& value) const { if(!has(name)) return false; value = operator[](name); return true; } bool Params::get(const std::string& name, std::vector& value) const { if(!has(name)) return false; std::vector v = operator[](name); value.assign(v.begin(),v.end()); return true; } bool Params::get(const std::string& name, std::vector& value) const { if(!has(name)) return false; std::vector v = operator[](name); value.assign(v.begin(),v.end()); return true; } bool Params::get(const std::string& name, std::vector& value) const { if(!has(name)) return false; std::vector v = operator[](name); value.assign(v.begin(),v.end()); return true; } */ Params::value_t Params::operator[](const Params::key_t& key) const { value_t v = getValue(*this, key); if (v.isNil()) { throw BadParameter("Params does not contain key: " + key, Here()); } return v; } Params::factory_map& Params::factories() { // Prevent static initialisation order fiasco by constructing the static // map on first use static factory_map factories_; return factories_; } Stream& operator<<(Stream& s, const Params& p) { encode(p, s); return s; } std::ostream& operator<<(std::ostream& s, const Params& p) { print(p, s); return s; } Params::value_t getValue(const Params& p, const Params::key_t& key) { return p.self_->get_(key); } void print(const Params& p, std::ostream& s) { p.self_->print_(s); } void encode(const Params& p, Stream& s) { p.self_->encode_(s); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/MapContent.h0000664000175000017500000000726215161702250020234 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Manuel Fuentes /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_MapContent_h #define eckit_MapContent_h #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class MapContent : public Content { protected: // -- Constructor MapContent(); MapContent(const ValueMap&); MapContent(Stream&); // -- Destructor ~MapContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override { Content::value(n); } void value(long long& n) const override { Content::value(n); } void value(double& n) const override { Content::value(n); } void value(std::string& n) const override { Content::value(n); } void value(Date& n) const override { Content::value(n); } void value(Time& n) const override { Content::value(n); } void value(DateTime& n) const override { Content::value(n); } void value(ValueList& n) const override { Content::value(n); } void value(ValueMap& n) const override; int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override { return -1; } int compareDouble(const DoubleContent&) const override { return -1; } int compareString(const StringContent&) const override { return -1; } int compareNil(const NilContent&) const override { return -1; } int compareList(const ListContent&) const override { return -1; } int compareMap(const MapContent&) const override; int compareDate(const DateContent&) const override { return 1; } int compareTime(const TimeContent&) const override { return 1; } int compareDateTime(const DateTimeContent&) const override { return 1; } int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; Value keys() const override; Value& element(const Value&) override; bool contains(const Value& key) const override; Value remove(const Value&) override; void print(std::ostream&) const override; void json(JSON&) const override; std::string typeName() const override { return "Map"; } bool isMap() const override { return true; } Content* clone() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- Members ValueMap value_; // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/OrderedMapContent.cc0000664000175000017500000001405115161702250021671 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/JSON.h" #include "eckit/value/OrderedMapContent.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec OrderedMapContent::classSpec_ = { &Content::classSpec(), "OrderedMapContent", }; Reanimator OrderedMapContent::reanimator_; OrderedMapContent::OrderedMapContent() {} OrderedMapContent::OrderedMapContent(const ValueMap& v, const ValueList& keys) : value_(v) { ASSERT(keys.size() == value_.size()); keys_ = keys; } OrderedMapContent::OrderedMapContent(Stream& s) : Content(s) { bool more; s >> more; while (more) { Value k(s); Value v(s); value_[k] = v; s >> more; } for (size_t i = 0; i < value_.size(); i++) { Value v(s); keys_.push_back(v); } } void OrderedMapContent::encode(Stream& s) const { Content::encode(s); for (ValueMap::const_iterator j = value_.begin(); j != value_.end(); ++j) { s << true; s << (*j).first; s << (*j).second; } s << false; for (size_t i = 0; i < value_.size(); i++) { s << keys_[i]; } } OrderedMapContent::~OrderedMapContent() {} void OrderedMapContent::value(ValueMap& v) const { v.clear(); for (ValueList::const_iterator j = keys_.begin(); j != keys_.end(); ++j) { v[(*j)] = value(*j); } } const Value& OrderedMapContent::value(const Value& key) const { ValueMap::const_iterator j = value_.find(key); ASSERT(j != value_.end()); return (*j).second; } Value OrderedMapContent::keys() const { return keys_; } Value OrderedMapContent::remove(const Value& key) { Value result = value_[key]; value_.erase(key); auto it = std::find(keys_.begin(), keys_.end(), key); if (it != keys_.end()) { keys_.erase(it); } return result; } Value& OrderedMapContent::element(const Value& key) { if (value_.find(key) == value_.end()) { // key is new so add too order list keys_.push_back(key); } return value_[key]; } bool OrderedMapContent::contains(const Value& key) const { return value_.find(key) != value_.end(); } int OrderedMapContent::compare(const Content& other) const { return -other.compareOrderedMap(*this); } int OrderedMapContent::compareOrderedMap(const OrderedMapContent& other) const { int b = 1; const ValueList* shorter = &keys_; const ValueList* longer = &other.keys_; bool swap = keys_.size() > other.keys_.size(); if (swap) { std::swap(shorter, longer); b = -1; } // compare the keys in order ValueList::const_iterator jc = (*longer).begin(); for (ValueList::const_iterator j = shorter->begin(); j != shorter->end(); ++j, ++jc) { if (*j == *jc) { continue; } return (*j < *jc) ? -b : b; } if (keys_.size() != other.keys_.size()) { return -b; } // the map with more elements is larger // all keys are equal and in same order, compare now the values jc = (*longer).begin(); for (ValueList::const_iterator j = shorter->begin(); j != shorter->end(); ++j, ++jc) { const Value& k = *j; const Value& left = value_.at(k); const Value& right = other.value_.at(k); if (left == right) { continue; } return (left < right) ? -b : b; } return 0; // all keys and values are the same and in same order } void OrderedMapContent::print(std::ostream& s) const { s << '{'; for (ValueList::const_iterator j = keys_.begin(); j != keys_.end(); ++j) { if (j != keys_.begin()) { s << " , "; } s << *j; s << " => "; s << value(*j); } s << '}'; } void OrderedMapContent::json(JSON& s) const { s.startObject(); for (ValueList::const_iterator j = keys_.begin(); j != keys_.end(); ++j) { s << *j; s << value(*j); } s.endObject(); } Content* OrderedMapContent::clone() const { OrderedMapContent* m = new OrderedMapContent(); for (const auto& key : keys_) { m->element(key.clone()) = value(key).clone(); } return m; } Content* OrderedMapContent::add(const Content& other) const { return other.addOrderedMap(*this); } Content* OrderedMapContent::sub(const Content& other) const { return other.subOrderedMap(*this); } Content* OrderedMapContent::mul(const Content& other) const { return other.mulOrderedMap(*this); } Content* OrderedMapContent::div(const Content& other) const { return other.divOrderedMap(*this); } Content* OrderedMapContent::mod(const Content& other) const { return other.modOrderedMap(*this); } void OrderedMapContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { size_t n = depth; while (n-- > 0) { out << ' '; } } out << "{"; const char* sep = "\n"; for (ValueList::const_iterator j = keys_.begin(); j != keys_.end(); ++j) { out << sep; (*j).dump(out, depth + 3); out << ": "; value(*j).dump(out, depth + 3, false); sep = ",\n"; } if (!value_.empty()) { out << '\n'; size_t n = depth; while (n-- > 0) { out << ' '; } } out << "}"; } void OrderedMapContent::hash(Hash& h) const { // Same as for MapContent. This is on purpose. for (auto v : value_) { v.first.hash(h); v.second.hash(h); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/Expression.h0000664000175000017500000001240515161702250020316 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Expression.h // Baudouin Raoult - ECMWF Oct 96 #ifndef Expression_H #define Expression_H #include #include #include "eckit/value/Value.h" //==================================================================== inline const char* opname(const std::negate&) { return "-"; } inline const char* opname(const std::multiplies&) { return "*"; } inline const char* opname(const std::divides&) { return "/"; } inline const char* opname(const std::modulus&) { return "%"; } inline const char* opname(const std::plus&) { return "+"; } inline const char* opname(const std::minus&) { return "-"; } inline const char* opname(const std::greater&) { return ">"; } inline const char* opname(const std::equal_to&) { return "=="; } inline const char* opname(const std::less&) { return "<"; } inline const char* opname(const std::greater_equal&) { return ">="; } inline const char* opname(const std::less_equal&) { return "<="; } inline const char* opname(const std::not_equal_to&) { return "!="; } inline const char* opname(const std::logical_not&) { return "!"; } inline const char* opname(const std::logical_and&) { return "&&"; } inline const char* opname(const std::logical_or&) { return "||"; } //==================================================================== class EvalError : public eckit::Exception { public: EvalError(const std::string& s) : Exception(std::string("EvalError: ") + s) {} }; template class Expression { virtual void print(std::ostream&) const = 0; public: Expression() = default; Expression(const Expression&) = delete; Expression& operator=(const Expression&) = delete; Expression(Expression&&) = delete; Expression& operator=(Expression&&) = delete; virtual eckit::Value eval(T&) const = 0; virtual ~Expression() {} friend std::ostream& operator<<(std::ostream& s, const Expression& c) { c.print(s); return s; } }; template class CondUnary : public Expression { std::unique_ptr > cond_; void print(std::ostream& s) const override { s << opname(T()) << '(' << *cond_ << ')'; } public: CondUnary(Expression* cond) : cond_(cond) {} ~CondUnary() override {} virtual eckit::Value eval(U& task) const { return T()(cond_->eval(task)); } }; template class CondBinary : public Expression { std::unique_ptr > left_; std::unique_ptr > right_; void print(std::ostream& s) const override { s << '(' << *left_ << ' ' << opname(T()) << ' ' << *right_ << ')'; } public: CondBinary(Expression* left, Expression* right) : left_(left), right_(right) {} ~CondBinary() override {} eckit::Value eval(U& task) const; }; template inline eckit::Value CondBinary::eval(U& task) const { return T()(left_->eval(task), right_->eval(task)); } template class StringExpression : public Expression { std::string str_; void print(std::ostream& s) const override { s << str_; } public: StringExpression(const std::string& s) : str_(s) {} ~StringExpression() override {} virtual eckit::Value eval(T&) const { return eckit::Value(str_); } }; template class NumberExpression : public Expression { long long value_; void print(std::ostream& s) const override { s << value_; } protected: long long value() const { return value_; } public: NumberExpression(long long n) : value_(n) {} ~NumberExpression() override {} virtual eckit::Value eval(T&) const { return eckit::Value(value_); } }; template class ListExpression : public Expression { std::vector*> v_; void print(std::ostream& s) const override; public: ListExpression(); ListExpression(const std::vector*>& v) : v_(v) {} ~ListExpression() override; virtual eckit::Value eval(T&) const; }; template ListExpression::~ListExpression() { for (size_t i = 0; i < v_.size(); i++) { delete v_[i]; } } template void ListExpression::print(std::ostream& s) const { s << '['; for (size_t i = 0; i < v_.size(); i++) { if (i) { s << ','; } if (v_[i]) { s << *v_[i]; } else { s << "(null)"; } } s << ']'; } template eckit::Value ListExpression::eval(T& t) const { std::vector v; for (size_t i = 0; i < v_.size(); i++) { if (v_[i]) { v.push_back(v_[i]->eval(t)); } } return v; } #endif eckit-2.0.7/src/eckit/value/BoolContent.cc0000664000175000017500000000545615161702250020553 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/BoolContent.h" #include "eckit/log/JSON.h" #include "eckit/utils/Hash.h" #include "eckit/utils/Translator.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec BoolContent::classSpec_ = { &Content::classSpec(), "BoolContent", }; Reanimator BoolContent::reanimator_; BoolContent::BoolContent(bool l) : value_(l) {} BoolContent::BoolContent(Stream& s) : Content(s), value_(false) { s >> value_; } Content* BoolContent::clone() const { return new BoolContent(value_); } void BoolContent::encode(Stream& s) const { Content::encode(s); s << value_; } BoolContent::~BoolContent() {} void BoolContent::print(std::ostream& s) const { s << (value_ ? "true" : "false"); } void BoolContent::json(JSON& s) const { s << value_; } int BoolContent::compare(const Content& other) const { return -other.compareBool(*this); } int BoolContent::compareBool(const BoolContent& other) const { bool equal = !(value_ - other.value_); if (equal) { return 0; // both equal in value, hence 0 } if (!value_) { return -1; // this is false, hence smaller than other } return 1; // this is true, hence larger than other } void BoolContent::value(bool& l) const { l = value_; } void BoolContent::value(std::string& s) const { s = value_ ? "true" : "false"; } void BoolContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { while (depth-- > 0) { out << ' '; } } out << (value_ ? "true" : "false"); } void BoolContent::value(long long& l) const { l = value_; } void BoolContent::value(double& d) const { d = value_; } Content* BoolContent::add(const Content& other) const { return other.addBool(*this); } Content* BoolContent::sub(const Content& other) const { return other.subBool(*this); } Content* BoolContent::mul(const Content& other) const { return other.mulBool(*this); } Content* BoolContent::div(const Content& other) const { return other.divBool(*this); } Content* BoolContent::mod(const Content& other) const { return other.modBool(*this); } void BoolContent::hash(Hash& h) const { h.add(value_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/Properties.cc0000664000175000017500000000665215161702250020460 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/Properties.h" #include "eckit/log/JSON.h" #include "eckit/types/Types.h" #include "eckit/utils/MD5.h" #include "eckit/value/Params.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Properties::Properties() {} Properties::Properties(const property_t& value) { ASSERT(value.isOrderedMap() || value.isMap()); ValueMap value_map = value; for (ValueMap::const_iterator vit = value_map.begin(); vit != value_map.end(); ++vit) { props_[vit->first] = vit->second; } } Properties::Properties(Stream& s) { s >> props_; } bool Properties::has(const key_t& k) const { return (props_.find(k) != props_.end()); } Properties::property_t Properties::get(const key_t& k) const { PropertyMap::const_iterator vit = props_.find(k); if (vit != props_.end()) { return (*vit).second; } return property_t(); // return Nil Value... } Properties& Properties::set(const key_t& k, const property_t& v) { props_[k] = v; return *this; } Properties& Properties::set(const key_t& k, const Properties& p) { ValueMap pmap; for (PropertyMap::const_iterator vit = p.props_.begin(); vit != p.props_.end(); ++vit) { pmap[vit->first] = vit->second; } props_[k] = Value::makeMap(pmap); return *this; } Properties& Properties::set(const Properties& p) { for (PropertyMap::const_iterator vit = p.props_.begin(); vit != p.props_.end(); ++vit) { props_[vit->first] = vit->second; } return *this; } bool Properties::remove(const key_t& k) { return props_.erase(k); } void Properties::hash(MD5& md5) const { for (PropertyMap::const_iterator vit = props_.begin(); vit != props_.end(); ++vit) { md5.add((*vit).first); /// @note below, we assume all Values translate to std::string, this needs more verification md5.add((*vit).second.as()); } } void Properties::json(JSON& s) const { s << props_; } void Properties::print(std::ostream& s) const { for (PropertyMap::const_iterator vit = props_.begin(); vit != props_.end(); ++vit) { s << "(" << (*vit).first << "," << (*vit).second << ")"; } } void Properties::encode(Stream& s) const { s << props_; } Properties::operator property_t() const { ValueMap vmap = Value::makeMap(); for (PropertyMap::const_iterator vit = props_.begin(); vit != props_.end(); ++vit) { vmap[vit->first] = vit->second; } return vmap; } //---------------------------------------------------------------------------------------------------------------------- Properties::property_t getValue(const Properties& p, const Properties::key_t& key) { return p.get(key); } void print(const Properties& p, std::ostream& s) { s << p; } void encode(const Properties& p, Stream& s) { s << p; } Params::Factory propertiesFactory; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/NumberContent.cc0000664000175000017500000001011415161702250021073 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/NumberContent.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/JSON.h" #include "eckit/utils/Translator.h" #include "eckit/value/DoubleContent.h" #include "eckit/utils/Hash.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class BadBoolConversion : public Exception { public: BadBoolConversion(const std::string& w) : Exception(std::string("Bad Bool Conversion: ") + w) {} }; //---------------------------------------------------------------------------------------------------------------------- ClassSpec NumberContent::classSpec_ = { &Content::classSpec(), "NumberContent", }; Reanimator NumberContent::reanimator_; NumberContent::NumberContent(long long l) : value_(l) {} NumberContent::NumberContent(Stream& s) : Content(s), value_(0) { s >> value_; } Content* NumberContent::clone() const { return new NumberContent(value_); } void NumberContent::encode(Stream& s) const { Content::encode(s); s << value_; } NumberContent::~NumberContent() {} void NumberContent::print(std::ostream& s) const { s << value_; } void NumberContent::json(JSON& s) const { s << value_; } int NumberContent::compare(const Content& other) const { return -other.compareNumber(*this); } int NumberContent::compareNumber(const NumberContent& other) const { long long dif = (value_ - other.value_); if (dif == 0) { return dif; } if (dif < 0) { return -1; } return 1; } int NumberContent::compareDouble(const DoubleContent& other) const { double dif = (value_ - other.value_); if (dif == 0) { return dif; } if (dif < 0) { return -1; } return 1; } void NumberContent::value(bool& b) const { if (value_ == 0) { b = false; } else { b = true; } } void NumberContent::value(long long& l) const { l = value_; } void NumberContent::value(double& l) const { l = value_; } void NumberContent::value(std::string& s) const { s = Translator()(value_); } Content* NumberContent::add(const Content& other) const { return other.addNumber(*this); } Content* NumberContent::addNumber(const NumberContent& other) const { return new NumberContent(other.value_ + value_); } Content* NumberContent::sub(const Content& other) const { return other.subNumber(*this); } Content* NumberContent::subNumber(const NumberContent& other) const { return new NumberContent(other.value_ - value_); } Content* NumberContent::mul(const Content& other) const { return other.mulNumber(*this); } Content* NumberContent::mod(const Content& other) const { return other.modNumber(*this); } Content* NumberContent::modNumber(const NumberContent& other) const { return new NumberContent(int(other.value_) % int(value_)); } Content* NumberContent::mulNumber(const NumberContent& other) const { return new NumberContent(other.value_ * value_); } Content* NumberContent::div(const Content& other) const { return other.divNumber(*this); } Content* NumberContent::divNumber(const NumberContent& other) const { return new NumberContent(other.value_ / value_); } Value NumberContent::negate() const { return Value(-value_); } void NumberContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { while (depth-- > 0) { out << ' '; } } out << "number(" << value_ << ")"; } void NumberContent::hash(Hash& h) const { h.add(value_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/OrderedMapContent.h0000664000175000017500000000765515161702250021547 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Manuel Fuentes /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_OrderedMapContent_h #define eckit_OrderedMapContent_h #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Hash; class OrderedMapContent : public Content { protected: // -- Constructor OrderedMapContent(); OrderedMapContent(const ValueMap&, const ValueList&); OrderedMapContent(Stream&); // -- Destructor ~OrderedMapContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override { Content::value(n); } void value(long long& n) const override { Content::value(n); } void value(double& n) const override { Content::value(n); } void value(std::string& n) const override { Content::value(n); } void value(Date& n) const override { Content::value(n); } void value(Time& n) const override { Content::value(n); } void value(DateTime& n) const override { Content::value(n); } void value(ValueList& n) const override { Content::value(n); } void value(ValueMap& n) const override; int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override { return -1; } int compareDouble(const DoubleContent&) const override { return -1; } int compareString(const StringContent&) const override { return -1; } int compareNil(const NilContent&) const override { return -1; } int compareList(const ListContent&) const override { return -1; } int compareMap(const MapContent&) const override { return -1; } int compareDate(const DateContent&) const override { return -1; } int compareTime(const TimeContent&) const override { return -1; } int compareDateTime(const DateTimeContent&) const override { return -1; } int compareOrderedMap(const OrderedMapContent&) const override; Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; Value keys() const override; Value& element(const Value&) override; bool contains(const Value& key) const override; Value remove(const Value&) override; void print(std::ostream&) const override; void json(JSON&) const override; std::string typeName() const override { return "OrderedMap"; } bool isMap() const override { return true; } bool isOrderedMap() const override { return true; } Content* clone() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- Methods const Value& value(const Value& key) const; // -- Members ValueMap value_; ValueList keys_; // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/ScopeParams.cc0000664000175000017500000000274015161702250020533 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/utils/StringTools.h" #include "eckit/value/ScopeParams.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ScopeParams::ScopeParams(const Params::key_t& scope_key, const Params& p) : scope_(scope_key + "."), p_(p) {} ScopeParams::ScopeParams(Stream& s) : p_(Params::decode(s)) { s >> scope_; } Params::value_t getValue(const ScopeParams& p, const Params::key_t& key) { if (StringTools::startsWith(key, p.scope_)) { return getValue(p.p_, key.substr(p.scope_.length())); } return Params::value_t(); } void print(const ScopeParams& p, std::ostream& s) { print(p.p_, s); } void encode(const ScopeParams& p, Stream& s) { s << p.p_; s << p.scope_; } Params::Factory scopeParamsFactory; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/TimeContent.h0000664000175000017500000000722115161702250020410 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Manuel Fuentes /// @author Tiago Quintino /// @date Jun 1997 #ifndef eckit_TimeContent_h #define eckit_TimeContent_h #include "eckit/value/Content.h" #include "eckit/value/Value.h" namespace eckit { class Hash; //---------------------------------------------------------------------------------------------------------------------- class TimeContent : public Content { protected: // -- Constructor TimeContent(const Time&); TimeContent(Stream&); // -- Destructor ~TimeContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override { Content::value(n); } void value(long long& n) const override { Content::value(n); } void value(double& n) const override { Content::value(n); } void value(std::string& n) const override { Content::value(n); } void value(Date& n) const override { Content::value(n); } void value(Time& n) const override; void value(DateTime& n) const override { Content::value(n); } void value(ValueList& n) const override { Content::value(n); } void value(ValueMap& n) const override { Content::value(n); } int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override { return -1; } int compareDouble(const DoubleContent&) const override { return -1; } int compareString(const StringContent&) const override { return -1; } int compareNil(const NilContent&) const override { return -1; } int compareList(const ListContent&) const override { return -1; } int compareMap(const MapContent&) const override { return -1; } int compareDate(const DateContent&) const override { return -1; } int compareTime(const TimeContent&) const override; int compareDateTime(const DateTimeContent&) const override { return 1; } int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; void print(std::ostream&) const override; void json(JSON&) const override; std::string typeName() const override { return "Time"; } bool isTime() const override { return true; } Content* clone() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: TimeContent(const TimeContent&); TimeContent& operator=(const TimeContent&); // -- Members Time value_; // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/Value.h0000664000175000017500000003574715161702250017251 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Manuel Fuentes /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_Value_h #define eckit_Value_h #include #include "eckit/types/Date.h" #include "eckit/types/DateTime.h" #include "eckit/value/Content.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// Known issues /// ============ /// /// NOTE: Objectives included not breaking eckit (including the testsuite). Therefore the failing unit tests are /// commented with ///. /// /// 1. Value(const Time&) and Value(const DateTime&) are unimplemented, even though exposed in the header. /// /// 2. Length(val) or (Length)val fail due to an ambiguity in the Length constructor, which doesn't know whether to /// cast via Length or long long. The explicit ::as() cast doesn't exist. Same with the Date() constructor. /// /// The definition of operator= differs from the constructors, so that it is possible to do: /// /// Length len = val; /// /// 3. Value(bool) happily converts to a double, even explicitly with ::as(). See BoolContent.cc/h /// /// 4. Length() and Offset() are supposed to define a well-defined algebra (which avoids errors such as adding two /// offsets). Within a Value they are stored as raw long long types, and no checking is performed when casting. Any /// numeric value (including negative values) may be used silently as Length/Offsets. /// /// 5. unsigned long long values are stored in signed long longs, resulting in integer overflows at (potentially) half /// the anticipated value. /// /// 6. Cannot initialise (copy) one ValueList with another using the implicit casts. ValueList is a std::vector, and /// there are two ambiguous constructors with appropriate overloads provided by Value /// /// std::vector( std::vector& rhs ); /// std::vector( size_type n ); /// /// Either: /// /// - Assign using operator= /// - Explicitly cast using ::as() /// /// 7. Operators ==, >, <, >=, <= act on the memory address of the allocated internal Content object, and have nothing /// to do with the content of the value. The compare() method should be used instead. /// /// Value(true) != Value(true) /// /// 8. ECKIT-260 (FIXED) /// /// 9. On conversion falure Value(std::string> --> long long, zero is returned rather than an exception being thrown. /// /// 10. On conversion failure Value(std::string) --> double, a BadParameter exception is thrown, rather than a /// BadConversion exception as for all other conversion failures (except the integer case noted above). /// /// 11. Comparison operators for Value(Date), Value(Time), Value(DateTime) have the wrong sign in MapContent.h:66-68 /// /// 12. Equality testing for ValueMaps gives unintuitive results, as the maps contain copies of Values, and these will /// have differing memory addresses (see (7)). /// /// 13. operator[] discards constness of ValueMaps: /// /// ValueMap vm; /// const Value cv(vm); /// cv[10]; /// /// Log::info() << cv; // This gives {10 => (nil)} /// /// 14. Indexing ValueMaps with Value(bool) doesn't work. I suspect this is due to point (8). /// /// 15. Indexing Value(ValueList) with std::string, or Value(std::string) spuriously returns element zero, rather than /// hitting an assertion or throwing another exception. This is a result of Value(std::string) silently casting any /// string to zero if used as an integer. /// /// ValueList vl; /// vl.push_back(123); /// Value val(vl); /// /// Log::info() << val["hello"]; // This prints "123". /// /// 16. Similarly, Value(ValueList)[Value(bool)] returns element zero or one depending on the content type. /// /// 17. Value(ValueList)::contains() should probably return false, rather than an exception, if indexed with bools, /// floats, ... /// /// 18. Although they are otherwise freely interconvertible, the arithmetic operators don't work between Number and /// Double types. /// /// 19. Although the mod() function is implemented on NumberContent, the corresponding modNumber() routine is not, and /// so BadOperator is returned incorrectly. /// /// More generally, the modulus functionality does not seem to be implemented for ANY of the Value() types. /// /// 20. Subtraction operators for Value(Date) have a sign error. A later date minus a newer date should be positive. /// /// 21. ECKIT-265 (FIXED) /// /// 22. The Value::head() and Value::tail() members make an entire copy of the contained ValueList before selecting /// and returning the head/tail element (which is copied). This will involve head allocations (and then /// deallocations during cleanup) for every contained element. /// /// -- This seems crazy. /// -- Accessing the first element of a list shouldn't be O[N] in cpu and memory... /// /// 23. The head() and tail() methods are clearly intended to be used on Value(ValueList) types, but if called on any /// other Value(X), it will create a single element list, and then return the first (and only) element. /// /// 24. The tail() method returns the entire list, excluding the first element (or an empty Value(), not an empty list, /// if there are no remaining elements). /// /// -- This is inconsistent with the naming, and the behaviour of head() /// -- It is closer to the car/cdr or first/rest functionality in lisp /// -- It should certainly at least return an empty list, not a nil value //---------------------------------------------------------------------------------------------------------------------- class Hash; class Length; class PathName; class JSON; class Value { public: // -- Contructors Value(); Value(bool); Value(int); Value(long); Value(long long); Value(unsigned int); Value(unsigned long); Value(unsigned long long); Value(double); Value(const std::string&); Value(const char*); Value(const Length&); Value(const Date&); Value(const Time&); Value(const DateTime&); Value(const PathName&); Value(Stream&); Value(const ValueList&); Value(const ValueMap&); // -- Copy Value(const Value&); Value& operator=(const Value&); // -- Destructor ~Value(); // -- Operators /// Explicitly cast value to the given type. For list of supported types, see the definitions of the /// member function value() in eckit/value/Content.h template T as() const { T r; content_->value(r); return r; } operator short() const { long long l; content_->value(l); return l; } operator unsigned short() const { long long l; content_->value(l); return l; } operator int() const { long long l; content_->value(l); return l; } operator unsigned int() const { long long l; content_->value(l); return l; } operator long() const { long long l; content_->value(l); return l; } operator unsigned long() const { long long l; content_->value(l); return l; } operator long long() const { long long l; content_->value(l); return l; } operator unsigned long long() const { long long l; content_->value(l); return l; } operator double() const { double d; content_->value(d); return d; } operator bool() const { bool d; content_->value(d); return d; } operator std::string() const { std::string s; content_->value(s); return s; } operator PathName() const { std::string s; content_->value(s); return s; } operator Date() const { Date d; content_->value(d); return d; } operator Time() const { Time t; content_->value(t); return t; } operator DateTime() const { DateTime d; content_->value(d); return d; } operator Length() const { long long l; content_->value(l); return l; } operator Offset() const { long long l; content_->value(l); return l; } operator ValueList() const; operator ValueMap() const; bool operator<(const Value& v) const { return *content_ < *(v.content_); } bool operator==(const Value& v) const { return *content_ == *(v.content_); } bool operator>(const Value& v) const { return v < *this; } bool operator!=(const Value& v) const { return !(*this == v); } bool operator>=(const Value& v) const { return !(*this < v); } bool operator<=(const Value& v) const { return !(v < *this); } Value operator+(const Value&) const; Value& operator+=(const Value&); Value operator-() const; Value operator-(const Value&) const; Value& operator-=(const Value&); Value operator*(const Value&) const; Value& operator*=(const Value&); Value operator/(const Value&) const; Value& operator/=(const Value&); Value operator%(const Value&) const; Value& operator%=(const Value&); Value operator[](const char*) const; Value operator[](const std::string&) const; Value operator[](const Value&) const; Value operator[](int) const; Value& operator[](const char*); Value& operator[](const std::string&); Value& operator[](const Value&); Value& operator[](int); Value keys() const; size_t size() const; std::ostream& dump(std::ostream& out, size_t depth = 0, bool indent = true) const; std::string typeName() const; void hash(eckit::Hash&) const; public: bool contains(const char*) const; bool contains(const std::string&) const; bool contains(const Value&) const; bool contains(int) const; Value& element(const Value&); Value element(const Value&) const; Value remove(const Value&); void append(const Value&); // List append // -- Methods int compare(const Value& v) const { return content_->compare(*(v.content_)); } bool isNil() const { return content_->isNil(); } bool isNumber() const { return content_->isNumber(); } bool isBool() const { return content_->isBool(); } bool isDouble() const { return content_->isDouble(); } bool isString() const { return content_->isString(); } bool isList() const { return content_->isList(); } bool isMap() const { return content_->isMap(); } bool isDate() const { return content_->isDate(); } bool isTime() const { return content_->isTime(); } bool isDateTime() const { return content_->isDateTime(); } bool isOrderedMap() const { return content_->isOrderedMap(); } Value tail() const; Value head() const; Value clone() const; bool shared() const; // Ensure that value is not shared // -- Class Methods static Value makeList(); static Value makeList(const Value&); static Value makeList(const ValueList&); static Value makeMap(); static Value makeMap(const ValueMap&); static Value makeOrderedMap(); static Value makeOrderedMap(const ValueMap&, const ValueList&); protected: Value(Content*); private: // members Content* content_; private: // methods void json(JSON& s) const { s << *content_; } void print(std::ostream& s) const { s << *content_; } void encode(Stream& s) const { s << *content_; } void update(); friend JSON& operator<<(JSON& s, const Value& v) { v.json(s); return s; } friend std::ostream& operator<<(std::ostream& s, const Value& v) { v.print(s); return s; } friend Stream& operator<<(Stream& s, const Value& v) { v.encode(s); return s; } friend class Content; }; //---------------------------------------------------------------------------------------------------------------------- template Value toValue(const T& v) { return Value(v); } template Value toValue(const std::set& l) { ValueList r; r.reserve(l.size()); for (typename std::set::const_iterator j = l.begin(); j != l.end(); ++j) { r.push_back(toValue(*j)); } return Value::makeList(r); } template Value toValue(const std::list& l) { ValueList r; r.reserve(l.size()); for (typename std::list::const_iterator j = l.begin(); j != l.end(); ++j) { r.push_back(toValue(*j)); } return Value::makeList(r); } template Value toValue(const std::vector& l) { ValueList r; r.reserve(l.size()); for (typename std::vector::const_iterator j = l.begin(); j != l.end(); ++j) { r.push_back(toValue(*j)); } return Value::makeList(r); } template Value toValue(const std::pair& v) { ValueList r; r.push_back(v.first); r.push_back(v.second); return Value::makeList(r); } template Value toValue(const std::map& l) { ValueMap r; for (typename std::map::const_iterator j = l.begin(); j != l.end(); ++j) { r[toValue((*j).first)] = toValue((*j).second); } return Value::makeMap(r); } //---------------------------------------------------------------------------------------------------------------------- template void fromValue(T& v, const Value& value) { v = T(value); } template void fromValue(std::vector& v, const Value& value) { v.clear(); for (size_t i = 0; i < value.size(); ++i) { T tmp; fromValue(tmp, value[i]); v.push_back(tmp); } } template void fromValue(std::pair& v, const Value& value) { ASSERT(value.size() == 2); fromValue(v.first, value[0]); fromValue(v.second, value[1]); } template void fromValue(std::set& v, const Value& value) { v.clear(); for (size_t i = 0; i < value.size(); ++i) { T tmp; fromValue(tmp, value[i]); v.insert(tmp); } } template void fromValue(std::map& v, const Value& value) { v.clear(); Value keys = value.keys(); for (size_t i = 0; i < keys.size(); ++i) { Value k = keys[i]; K key; fromValue(key, k); fromValue(v[key], value[k]); } } //---------------------------------------------------------------------------------------------------------------------- template <> struct VectorPrintSelector { using selector = VectorPrintSimple; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/value/StringContent.h0000664000175000017500000000726415161702250020767 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Manuel Fuentes /// @author Tiago Quintino /// @date Jun 97 #ifndef eckit_StringContent_h #define eckit_StringContent_h #include "eckit/value/Content.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Hash; class StringContent : public Content { protected: // -- Constructor StringContent(const std::string&); StringContent(const char*); StringContent(Stream&); // -- Destructor ~StringContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override; void value(long long& n) const override; void value(double& n) const override; void value(std::string& n) const override; void value(Date& n) const override { Content::value(n); } void value(Time& n) const override { Content::value(n); } void value(DateTime& n) const override { Content::value(n); } void value(ValueList& n) const override { Content::value(n); } void value(ValueMap& n) const override { Content::value(n); } int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override { return -1; } int compareDouble(const DoubleContent&) const override { return -1; } int compareString(const StringContent&) const override; int compareNil(const NilContent&) const override { return 1; } int compareList(const ListContent&) const override { return 1; } int compareMap(const MapContent&) const override { return 1; } int compareDate(const DateContent&) const override { return 1; } int compareTime(const TimeContent&) const override { return 1; } int compareDateTime(const DateTimeContent&) const override { return 1; } int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; Content* addString(const StringContent&) const override; void print(std::ostream&) const override; void json(JSON&) const override; std::string typeName() const override { return "String"; } bool isString() const override { return true; } Content* clone() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- No copy allowed StringContent(const StringContent&); StringContent& operator=(const StringContent&); // -- Members std::string value_; static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/NilContent.cc0000664000175000017500000000516315161702250020375 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/NilContent.h" #include "eckit/log/JSON.h" #include "eckit/value/Value.h" #include "eckit/utils/Hash.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec NilContent::classSpec_ = { &Content::classSpec(), "NilContent", }; Reanimator NilContent::reanimator_; NilContent::NilContent() {} NilContent::NilContent(Stream& s) : Content(s) {} Content* NilContent::clone() const { return new NilContent(); } void NilContent::encode(Stream& s) const { Content::encode(s); } NilContent::~NilContent() {} void NilContent::value(ValueList& v) const { v = ValueList(); } void NilContent::print(std::ostream& out) const { out << "(nil)"; } void NilContent::json(JSON& s) const { s.null(); } int NilContent::compare(const Content& other) const { return -other.compareNil(*this); } int NilContent::compareNil(const NilContent&) const { return 0; // They're equals } Content* NilContent::add(const Content& other) const { return other.addNil(*this); } Content* NilContent::addNil(const NilContent&) const { return (Content*)this; } Content* NilContent::sub(const Content& other) const { return other.subNil(*this); } Content* NilContent::subNil(const NilContent&) const { return (Content*)this; } Content* NilContent::mul(const Content& other) const { return other.mulNil(*this); } Content* NilContent::mulNil(const NilContent&) const { return (Content*)this; } Content* NilContent::div(const Content& other) const { return other.divNil(*this); } Content* NilContent::divNil(const NilContent&) const { return (Content*)this; } Content* NilContent::mod(const Content& other) const { return other.divNil(*this); } bool NilContent::contains(const Value&) const { return false; } void NilContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { while (depth-- > 0) { out << ' '; } } out << "nil"; } void NilContent::hash(Hash& h) const { h.add(""); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/Content.h0000664000175000017500000002157315161702250017577 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Content.h // Manuel Fuentes - ECMWF Jun 96 #ifndef eckit_Content_h #define eckit_Content_h #include #include #include #include "eckit/memory/Counted.h" #include "eckit/serialisation/Streamable.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- // List here all the Content's class DoubleContent; class BoolContent; class NumberContent; class StringContent; class NilContent; class ListContent; class MapContent; class OrderedMapContent; class DateContent; class TimeContent; class DateTimeContent; class Date; class Time; class DateTime; class Value; class JSON; class Hash; using ValueList = std::vector; using ValueMap = std::map; //---------------------------------------------------------------------------------------------------------------------- // Assuptions for comparisons: // Nil < Number < String < List class Content : public Counted, public Streamable { public: // Double-dispatching on subclasses // Needs a compare??? for every new subclass virtual int compareBool(const BoolContent&) const; virtual int compareNumber(const NumberContent&) const; virtual int compareDouble(const DoubleContent&) const; virtual int compareString(const StringContent&) const; virtual int compareNil(const NilContent&) const; virtual int compareList(const ListContent&) const; virtual int compareMap(const MapContent&) const; virtual int compareDate(const DateContent&) const; virtual int compareTime(const TimeContent&) const; virtual int compareDateTime(const DateTimeContent&) const; virtual int compareOrderedMap(const OrderedMapContent&) const; // Double-dispatching on subclasses for addition // Needs an add??? for every new subclass virtual Content* addNumber(const NumberContent&) const; virtual Content* addBool(const BoolContent&) const; virtual Content* addDouble(const DoubleContent&) const; virtual Content* addString(const StringContent&) const; virtual Content* addNil(const NilContent&) const; virtual Content* addList(const ListContent&) const; virtual Content* addMap(const MapContent&) const; virtual Content* addDate(const DateContent&) const; virtual Content* addTime(const TimeContent&) const; virtual Content* addDateTime(const DateTimeContent&) const; virtual Content* addOrderedMap(const OrderedMapContent&) const; virtual Content* subNumber(const NumberContent&) const; virtual Content* subDouble(const DoubleContent&) const; virtual Content* subBool(const BoolContent&) const; virtual Content* subString(const StringContent&) const; virtual Content* subNil(const NilContent&) const; virtual Content* subList(const ListContent&) const; virtual Content* subMap(const MapContent&) const; virtual Content* subDate(const DateContent&) const; virtual Content* subTime(const TimeContent&) const; virtual Content* subDateTime(const DateTimeContent&) const; virtual Content* subOrderedMap(const OrderedMapContent&) const; virtual Content* mulNumber(const NumberContent&) const; virtual Content* mulDouble(const DoubleContent&) const; virtual Content* mulBool(const BoolContent&) const; virtual Content* mulString(const StringContent&) const; virtual Content* mulNil(const NilContent&) const; virtual Content* mulList(const ListContent&) const; virtual Content* mulMap(const MapContent&) const; virtual Content* mulDate(const DateContent&) const; virtual Content* mulTime(const TimeContent&) const; virtual Content* mulDateTime(const DateTimeContent&) const; virtual Content* mulOrderedMap(const OrderedMapContent&) const; virtual Content* divNumber(const NumberContent&) const; virtual Content* divDouble(const DoubleContent&) const; virtual Content* divBool(const BoolContent&) const; virtual Content* divString(const StringContent&) const; virtual Content* divNil(const NilContent&) const; virtual Content* divList(const ListContent&) const; virtual Content* divMap(const MapContent&) const; virtual Content* divDate(const DateContent&) const; virtual Content* divTime(const TimeContent&) const; virtual Content* divDateTime(const DateTimeContent&) const; virtual Content* divOrderedMap(const OrderedMapContent&) const; virtual Content* modNumber(const NumberContent&) const; virtual Content* modDouble(const DoubleContent&) const; virtual Content* modBool(const BoolContent&) const; virtual Content* modString(const StringContent&) const; virtual Content* modNil(const NilContent&) const; virtual Content* modList(const ListContent&) const; virtual Content* modMap(const MapContent&) const; virtual Content* modDate(const DateContent&) const; virtual Content* modTime(const TimeContent&) const; virtual Content* modDateTime(const DateTimeContent&) const; virtual Content* modOrderedMap(const OrderedMapContent&) const; virtual void hash(Hash&) const = 0; protected: // -- Constructor Content(); Content(Stream&); // -- Destructor ~Content() override; // -- Operators bool operator==(const Content&) const; bool operator<(const Content&) const; // void *operator new(size_t s) { return MemoryPool::fastAllocate(s);} // void operator delete(void* p) { MemoryPool::fastDeallocate(p); } // -- Methods virtual void value(bool&) const; virtual void value(long long&) const; virtual void value(double&) const; virtual void value(std::string&) const; virtual void value(Date&) const; virtual void value(Time&) const; virtual void value(DateTime&) const; virtual void value(ValueList&) const; virtual void value(ValueMap&) const; Content* operator+(const Content&) const; Content* operator-(const Content&) const; Content* operator*(const Content&) const; Content* operator/(const Content&) const; // -- Methods virtual void print(std::ostream&) const = 0; virtual void dump(std::ostream&, size_t, bool indent = true) const = 0; virtual std::string typeName() const = 0; virtual void json(JSON&) const = 0; virtual Content* clone() const = 0; virtual bool isNil() const { return false; } virtual bool isNumber() const { return false; } virtual bool isBool() const { return false; } virtual bool isDouble() const { return false; } virtual bool isString() const { return false; } virtual bool isList() const { return false; } virtual bool isMap() const { return false; } virtual bool isDate() const { return false; } virtual bool isTime() const { return false; } virtual bool isDateTime() const { return false; } virtual bool isOrderedMap() const { return false; } virtual bool contains(const Value&) const; virtual Value& element(const Value&); virtual Value remove(const Value&); virtual void append(const Value&); virtual Value keys() const; virtual Value negate() const; virtual size_t size() const; // -- Overridden methods // From Streamble void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } private: // -- No copy allowed Content(const Content&); Content& operator=(const Content&); // -- Class members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Methods virtual int compare(const Content&) const = 0; virtual Content* add(const Content&) const = 0; virtual Content* sub(const Content&) const = 0; virtual Content* mul(const Content&) const = 0; virtual Content* div(const Content&) const = 0; virtual Content* mod(const Content&) const = 0; void badConversion(const std::string&) const; void badComparison(const std::string&) const; void badOperator(const std::string&, const std::string&) const; // -- Friends friend std::ostream& operator<<(std::ostream& s, const Content& content) { content.print(s); return s; } friend JSON& operator<<(JSON& s, const Content& content) { content.json(s); return s; } friend class Value; }; template <> Streamable* Reanimator::ressucitate(Stream& s) const #ifdef IBM { return 0; } #else ; #endif //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif // eckit_Content_h eckit-2.0.7/src/eckit/value/TimeContent.cc0000664000175000017500000000475215161702250020554 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/TimeContent.h" #include "eckit/log/JSON.h" #include "eckit/utils/Hash.h" #include "eckit/value/NumberContent.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec TimeContent::classSpec_ = { &Content::classSpec(), "TimeContent", }; Reanimator TimeContent::reanimator_; TimeContent::TimeContent(const Time& d) : value_(d) {} TimeContent::TimeContent(Stream& s) : Content(s) { std::string dd; s >> dd; value_ = Time(dd); } void TimeContent::encode(Stream& s) const { Content::encode(s); std::string dd = value_; s << dd; } TimeContent::~TimeContent() {} Content* TimeContent::clone() const { return new TimeContent(value_); } void TimeContent::print(std::ostream& s) const { s << value_; } void TimeContent::json(JSON& s) const { s << std::string(value_); } int TimeContent::compare(const Content& other) const { return -other.compareTime(*this); } int TimeContent::compareTime(const TimeContent& other) const { if (value_ == other.value_) { return 0; } return (value_ < other.value_) ? -1 : 1; } void TimeContent::value(Time& d) const { d = value_; } Content* TimeContent::add(const Content& other) const { return other.addTime(*this); } Content* TimeContent::sub(const Content& other) const { return other.subTime(*this); } Content* TimeContent::mul(const Content& other) const { return other.mulTime(*this); } Content* TimeContent::div(const Content& other) const { return other.divTime(*this); } Content* TimeContent::mod(const Content& other) const { return other.modTime(*this); } void TimeContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { while (depth-- > 0) { out << ' '; } } out << "time(" << value_ << ")"; } void TimeContent::hash(Hash& h) const { value_.hash(h); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/Value.cc0000664000175000017500000001740615161702250017377 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/Value.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/Length.h" #include "eckit/value/BoolContent.h" #include "eckit/value/DateContent.h" #include "eckit/value/DateTimeContent.h" #include "eckit/value/DoubleContent.h" #include "eckit/value/ListContent.h" #include "eckit/value/MapContent.h" #include "eckit/value/NilContent.h" #include "eckit/value/NumberContent.h" #include "eckit/value/OrderedMapContent.h" #include "eckit/value/StringContent.h" #include "eckit/value/TimeContent.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { namespace { class Nil : public NilContent { public: Nil() { attach(); } // the only instance of Nil (below) *will* be leaked at_exit() }; static Nil* nil = nullptr; // must be a pointer, so we control when is created to respect order of destruction at_exit() static Nil* nill() { if (!nil) { nil = new Nil(); } return nil; } } // namespace //---------------------------------------------------------------------------------------------------------------------- Value::Value() : content_(nill()) { content_->attach(); } Value::Value(int l) : content_(new NumberContent(l)) { content_->attach(); } Value::Value(long long l) : content_(new NumberContent(l)) { content_->attach(); } Value::Value(unsigned long long l) : content_(new NumberContent(l)) { content_->attach(); } Value::Value(unsigned long l) : content_(new NumberContent(l)) { content_->attach(); } Value::Value(unsigned int l) : content_(new NumberContent(l)) { content_->attach(); } Value::Value(long l) : content_(new NumberContent(l)) { content_->attach(); } Value::Value(bool l) : content_(new BoolContent(l)) { content_->attach(); } Value::Value(double l) : content_(new DoubleContent(l)) { content_->attach(); } Value::Value(const std::string& s) : content_(new StringContent(s)) { content_->attach(); } Value::Value(const char* s) : content_(new StringContent(s)) { content_->attach(); } Value::Value(const Length& l) : content_(new NumberContent(l)) { content_->attach(); } Value::Value(const PathName& p) : content_(new StringContent(p.asString())) { content_->attach(); } Value::Value(const Date& d) : content_(new DateContent(d)) { content_->attach(); } Value::Value(const Time& d) : content_(new TimeContent(d)) { content_->attach(); } Value::Value(const DateTime& d) : content_(new DateTimeContent(d)) { content_->attach(); } Value::Value(Stream& s) : content_(Reanimator::reanimate(s)) { ASSERT(content_); content_->attach(); } Value::~Value() { content_->detach(); } Value::Value(const Value& other) : content_(other.content_) { content_->attach(); } Value Value::clone() const { return Value(content_->clone()); } bool Value::shared() const { return content_->count() > 1; } Value& Value::operator=(const Value& other) { if (this == &other) { return *this; } Content* current = content_; content_ = other.content_; content_->attach(); current->detach(); return *this; } Value Value::operator+(const Value& v) const { return Value(content_->add(*(v.content_))); } Value& Value::operator+=(const Value& v) { *this = *this + v; return *this; } Value Value::operator-(const Value& v) const { return Value(content_->sub(*(v.content_))); } Value& Value::operator-=(const Value& v) { *this = *this - v; return *this; } Value Value::operator*(const Value& v) const { return Value(content_->mul(*(v.content_))); } Value& Value::operator*=(const Value& v) { *this = *this * v; return *this; } Value Value::operator/(const Value& v) const { return Value(content_->div(*(v.content_))); } Value& Value::operator/=(const Value& v) { *this = *this / v; return *this; } Value Value::operator%(const Value& v) const { return Value(content_->mod(*(v.content_))); } Value& Value::operator%=(const Value& v) { *this = *this % v; return *this; } Value Value::makeList() { return Value(new ListContent()); } Value Value::makeMap() { return Value(new MapContent()); } Value Value::makeOrderedMap() { return Value(new OrderedMapContent()); } Value Value::makeMap(const ValueMap& m) { return Value(new MapContent(m)); } Value Value::makeOrderedMap(const ValueMap& m, const ValueList& l) { return Value(new OrderedMapContent(m, l)); } Value Value::makeList(const Value& v) { return Value(new ListContent(v)); } Value Value::makeList(const ValueList& v) { return Value(new ListContent(v)); } Value::Value(const ValueList& v) : content_(new ListContent(v)) { content_->attach(); } Value::Value(const ValueMap& m) : content_(new MapContent(m)) { content_->attach(); } Value::Value(Content* c) : content_(c) { content_->attach(); } Value Value::head() const { ValueList v; content_->value(v); return v.size() > 0 ? v[0] : Value(); } Value Value::tail() const { ValueList v; content_->value(v); if (v.size() > 1) { v.erase(v.begin()); return v; } return Value(); } Value::operator ValueList() const { ValueList v; content_->value(v); return v; } Value::operator ValueMap() const { ValueMap v; content_->value(v); return v; } bool Value::contains(const Value& key) const { return content_->contains(key); } bool Value::contains(const char* key) const { return content_->contains(key); } bool Value::contains(const std::string& key) const { return content_->contains(key); } bool Value::contains(int key) const { return content_->contains(key); } Value Value::operator-() const { return content_->negate(); } Value Value::keys() const { return content_->keys(); } size_t Value::size() const { return content_->size(); } std::ostream& Value::dump(std::ostream& out, size_t depth, bool indent) const { content_->dump(out, depth, indent); return out; } std::string Value::typeName() const { return content_->typeName(); } void Value::hash(Hash& h) const { return content_->hash(h); } Value Value::operator[](const char* key) const { return element(Value(key)); } Value Value::operator[](const std::string& key) const { return element(Value(key)); } Value Value::operator[](const Value& key) const { return element(key); } Value Value::operator[](int key) const { return element(Value(key)); } Value& Value::operator[](const char* key) { return element(Value(key)); } Value& Value::operator[](const std::string& key) { return element(Value(key)); } Value& Value::operator[](const Value& key) { return element(key); } Value& Value::operator[](int key) { return element(Value(key)); } Value& Value::element(const Value& key) { update(); return content_->element(key); } Value Value::element(const Value& key) const { return content_->element(key); } Value Value::remove(const Value& key) { update(); return content_->remove(key); } void Value::update() { if (content_->count() > 1) { Content* c = content_->clone(); c->attach(); content_->detach(); content_ = c; } } void Value::append(const Value& value) { update(); content_->append(value); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/Params.h0000664000175000017500000001145115161702250017402 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Florian Rathgeber /// @date July 2014 #ifndef eckit_value_Params_H #define eckit_value_Params_H #include #include "eckit/config/Parametrisation.h" #include "eckit/serialisation/Stream.h" #include "eckit/value/Value.h" //---------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------- /// Params provides a value-based behaviour for parametrisations of objects. /// A class `MyParams` to be wrapped into Params needs to implement: /// /// 1. A constructor from `Stream&` /// /// 2. A static method `className` returning a `const char*` /// /// 3. A `get` function taking a `Params::key_t` and returning a `Params::value_t`: /// /// Params::value_t get( const MyParams&, const Params::key_t& ) /// /// This function performs the lookup of a parameter by key. /// /// 4. A `print` function taking a `std::ostream&`: /// /// void print( const MyParams&, std::ostream& ) /// /// This function prints a representation of the parameters to the ostream. /// /// 5. An `encode` function taking a `Stream&`: /// /// void encode( const MyParams&, Stream& ) /// /// This function encodes the parameters to the Stream. /// /// If the free functions `get`, `print` and `encode` need access to private /// members of `MyParams`, they need to be declared as friend functions. /// /// In addition a `Params::Factory` needs to be initialised. class Params { struct Concept; public: // types using List = std::list; using key_t = std::string; using value_t = Value; struct BaseFactory { virtual ~BaseFactory() {} virtual Concept* build(Stream& s) = 0; }; using factory_t = BaseFactory*; template struct Factory : BaseFactory { Factory() { Params::registerFactory(T::className(), this); } Concept* build(Stream& s); }; public: // methods template explicit Params(const T& x) : self_(new Model(x)) {} Params(const Params& x) : self_(x.self_->copy_()) {} static void registerFactory(const std::string& name, factory_t f) { factories()[name] = f; } static factory_t& getFactory(const std::string& name) { return factories()[name]; } static Params build(const std::string& name, Stream& s) { return Params(getFactory(name)->build(s)); // returns Concept* } static Params decode(Stream& s) { std::string name; s >> name; return build(name, s); } ~Params() { delete self_; } Params& operator=(Params x) { std::swap(x.self_, this->self_); return *this; } bool has(const std::string& name) const; value_t operator[](const key_t& key) const; friend void print(const Params& p, std::ostream& s); friend void encode(const Params& p, Stream& s); friend value_t getValue(const Params& p, const key_t& key); private: // internal classes using factory_map = std::map; static factory_map& factories(); Params(Concept* _concept) : self_(_concept) {} struct Concept { virtual ~Concept() {} virtual Concept* copy_() const = 0; virtual value_t get_(const key_t& key) const = 0; virtual void print_(std::ostream& s) const = 0; virtual void encode_(Stream& s) const = 0; }; template struct Model : Concept { Model(T x) : data_(x) {} Model(Stream& s) : data_(s) {} virtual Concept* copy_() const { return new Model(data_); } virtual value_t get_(const key_t& key) const { return getValue(data_, key); } virtual void print_(std::ostream& s) const { print(data_, s); } virtual void encode_(Stream& s) const { s << T::className(); encode(data_, s); } T data_; }; private: // methods friend std::ostream& operator<<(std::ostream& s, const Params& p); friend Stream& operator<<(Stream& s, const Params& p); private: // members const Concept* self_; }; //---------------------------------------------------------------------------- template Params::Concept* Params::Factory::build(Stream& s) { return new Model(s); } //---------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/ScopeParams.h0000664000175000017500000000260215161702250020372 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Florian Rathgeber /// @date March 2015 #ifndef eckit_value_ScopeParams_H #define eckit_value_ScopeParams_H #include "eckit/value/Params.h" //---------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------- /// Wraps the parameters within a given scope class ScopeParams { public: // methods ScopeParams(const Params::key_t& scope_key, const Params& p); ScopeParams(Stream& s); static const char* className() { return "eckit::ScopeParams"; } private: // methods friend Params::value_t getValue(const ScopeParams& p, const Params::key_t& key); friend void print(const ScopeParams& p, std::ostream& s); friend void encode(const ScopeParams& p, Stream& s); private: // members Params::key_t scope_; Params p_; }; //---------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/DateContent.cc0000664000175000017500000000514315161702250020526 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/DateContent.h" #include "eckit/log/JSON.h" #include "eckit/utils/Hash.h" #include "eckit/value/NumberContent.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec DateContent::classSpec_ = { &Content::classSpec(), "DateContent", }; Reanimator DateContent::reanimator_; DateContent::DateContent(const Date& d) : value_(d) {} DateContent::DateContent(Stream& s) : Content(s) { std::string dd; s >> dd; value_ = Date(dd); } void DateContent::encode(Stream& s) const { Content::encode(s); std::string dd = value_; s << dd; } DateContent::~DateContent() {} Content* DateContent::clone() const { return new DateContent(value_); } void DateContent::print(std::ostream& s) const { s << value_; } void DateContent::json(JSON& s) const { s << std::string(value_); } int DateContent::compare(const Content& other) const { return -other.compareDate(*this); } int DateContent::compareDate(const DateContent& other) const { if (value_ == other.value_) { return 0; } return (value_ < other.value_) ? -1 : 1; } void DateContent::value(Date& d) const { d = value_; } Content* DateContent::add(const Content& other) const { return other.addDate(*this); } Content* DateContent::sub(const Content& other) const { return other.subDate(*this); } Content* DateContent::subDate(const DateContent& other) const { return new NumberContent(value_ - other.value_); } Content* DateContent::mul(const Content& other) const { return other.mulDate(*this); } Content* DateContent::div(const Content& other) const { return other.divDate(*this); } Content* DateContent::mod(const Content& other) const { return other.modDate(*this); } void DateContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { while (depth-- > 0) { out << ' '; } } out << "date(" << value_ << ")"; } void DateContent::hash(Hash& h) const { value_.hash(h); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/ListContent.h0000664000175000017500000000732315161702250020430 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Baudouin Raoult /// @author Manuel Fuentes #ifndef eckit_ListContent_h #define eckit_ListContent_h #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class ListContent : public Content { protected: // -- Constructor ListContent(); ListContent(const ValueList&); ListContent(const Value&); ListContent(Stream&); // -- Destructor ~ListContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override; void value(long long& n) const override; void value(double& n) const override; void value(std::string& n) const override; void value(Date& n) const override; void value(Time& n) const override; void value(DateTime& n) const override; void value(ValueList& n) const override; void value(ValueMap& n) const override { Content::value(n); } int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override { return -1; } int compareDouble(const DoubleContent&) const override { return -1; } int compareString(const StringContent&) const override { return -1; } int compareNil(const NilContent&) const override { return -1; } int compareList(const ListContent&) const override; int compareMap(const MapContent&) const override { return 1; } int compareDate(const DateContent&) const override { return 1; } int compareTime(const TimeContent&) const override { return 1; } int compareDateTime(const DateTimeContent&) const override { return 1; } int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; Content* addList(const ListContent&) const override; void print(std::ostream&) const override; void json(JSON&) const override; std::string typeName() const override { return "List"; } bool isList() const override { return true; } Value& element(const Value&) override; bool contains(const Value& key) const override; void append(const Value&) override; Content* clone() const override; size_t size() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: ListContent(const ListContent&); ListContent& operator=(const ListContent&); // -- Members ValueList value_; // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/DateTimeContent.cc0000664000175000017500000000514315161702250021345 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/DateTimeContent.h" #include "eckit/log/JSON.h" #include "eckit/value/NumberContent.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec DateTimeContent::classSpec_ = { &Content::classSpec(), "DateTimeContent", }; Reanimator DateTimeContent::reanimator_; DateTimeContent::DateTimeContent(const DateTime& d) : value_(d) {} DateTimeContent::DateTimeContent(Stream& s) : Content(s) { std::string dd; s >> dd; value_ = DateTime(dd); } void DateTimeContent::encode(Stream& s) const { Content::encode(s); std::string dd = value_; s << dd; } DateTimeContent::~DateTimeContent() {} Content* DateTimeContent::clone() const { return new DateTimeContent(value_); } void DateTimeContent::print(std::ostream& s) const { s << value_; } void DateTimeContent::json(JSON& s) const { s << std::string(value_); } int DateTimeContent::compare(const Content& other) const { return -other.compareDateTime(*this); } int DateTimeContent::compareDateTime(const DateTimeContent& other) const { if (value_ == other.value_) { return 0; } return (value_ < other.value_) ? -1 : 1; } void DateTimeContent::value(DateTime& d) const { d = value_; } Content* DateTimeContent::add(const Content& other) const { return other.addDateTime(*this); } Content* DateTimeContent::sub(const Content& other) const { return other.subDateTime(*this); } Content* DateTimeContent::mul(const Content& other) const { return other.mulDateTime(*this); } Content* DateTimeContent::div(const Content& other) const { return other.divDateTime(*this); } Content* DateTimeContent::mod(const Content& other) const { return other.modDateTime(*this); } void DateTimeContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { while (depth-- > 0) { out << ' '; } } out << "datetime(" << value_ << ")"; } void DateTimeContent::hash(Hash& h) const { value_.hash(h); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/CompositeParams.h0000664000175000017500000000317115161702250021265 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Florian Rathgeber /// @date March 2015 #ifndef eckit_value_CompositeParams_H #define eckit_value_CompositeParams_H #include "eckit/value/Params.h" //---------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------- class CompositeParams { public: // methods CompositeParams(); CompositeParams(const Params::List&); CompositeParams(Stream&); CompositeParams& push_front(const Params& p); CompositeParams& push_back(const Params& p); CompositeParams& pop_front() { plist_.pop_front(); return *this; } CompositeParams& pop_back() { plist_.pop_back(); return *this; } static const char* className() { return "eckit::CompositeParams"; } private: // methods friend Params::value_t getValue(const CompositeParams& p, const Params::key_t& key); friend void print(const CompositeParams& p, std::ostream& s); friend void encode(const CompositeParams& p, Stream& s); private: // members Params::List plist_; }; //---------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/DateTimeContent.h0000664000175000017500000000725415161702250021214 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Baudouin Raoult /// @author Manuel Fuentes #ifndef eckit_DateTimeContent_h #define eckit_DateTimeContent_h #include "eckit/value/Content.h" #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class DateTimeContent : public Content { protected: // -- Constructor DateTimeContent(const DateTime&); DateTimeContent(Stream&); // -- Destructor ~DateTimeContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override { Content::value(n); } void value(long long& n) const override { Content::value(n); } void value(double& n) const override { Content::value(n); } void value(std::string& n) const override { Content::value(n); } void value(Date& n) const override { Content::value(n); } void value(Time& n) const override { Content::value(n); } void value(DateTime& n) const override; void value(ValueList& n) const override { Content::value(n); } void value(ValueMap& n) const override { Content::value(n); } int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override { return -1; } int compareDouble(const DoubleContent&) const override { return -1; } int compareString(const StringContent&) const override { return -1; } int compareNil(const NilContent&) const override { return -1; } int compareList(const ListContent&) const override { return -1; } int compareMap(const MapContent&) const override { return -1; } int compareDate(const DateContent&) const override { return -1; } int compareTime(const TimeContent&) const override { return -1; } int compareDateTime(const DateTimeContent&) const override; int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; void print(std::ostream&) const override; void json(JSON&) const override; std::string typeName() const override { return "DateTime"; } bool isDate() const override { return true; } Content* clone() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: DateTimeContent(const DateTimeContent&); DateTimeContent& operator=(const DateTimeContent&); // -- Members DateTime value_; // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/NumberContent.h0000664000175000017500000001033115161702250020736 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Baudouin Raoult /// @author Manuel Fuentes #ifndef eckit_NumberContent_h #define eckit_NumberContent_h #include "eckit/value/Content.h" #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class NumberContent : public Content { protected: // -- Constructor NumberContent(long long); NumberContent(Stream&); // -- Destructor ~NumberContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override; void value(long long& n) const override; void value(double& n) const override; void value(std::string& n) const override; void value(Date& n) const override { Content::value(n); } void value(Time& n) const override { Content::value(n); } void value(DateTime& n) const override { Content::value(n); } void value(ValueList& n) const override { Content::value(n); } void value(ValueMap& n) const override { Content::value(n); } int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override; int compareDouble(const DoubleContent&) const override; int compareString(const StringContent&) const override { return 1; } int compareNil(const NilContent&) const override { return 1; } int compareList(const ListContent&) const override { return 1; } int compareMap(const MapContent&) const override { return 1; } int compareDate(const DateContent&) const override { return 1; } int compareTime(const TimeContent&) const override { return 1; } int compareDateTime(const DateTimeContent&) const override { return 1; } int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; Content* addNumber(const NumberContent&) const override; Content* subNumber(const NumberContent&) const override; Content* mulNumber(const NumberContent&) const override; Content* divNumber(const NumberContent&) const override; Content* modNumber(const NumberContent&) const override; Value negate() const override; // Content* addDouble(const DoubleContent&) const override; // Content* subDouble(const DoubleContent&) const override; // Content* mulDouble(const DoubleContent&) const override; // Content* divDouble(const DoubleContent&) const override; void print(std::ostream&) const override; void json(JSON&) const override; std::string typeName() const override { return "Number"; } bool isNumber() const override { return true; } Content* clone() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: NumberContent(const NumberContent&); NumberContent& operator=(const NumberContent&); // -- Members long long value_; // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; friend class DateContent; friend class DoubleContent; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/ListContent.cc0000664000175000017500000001205515161702250020564 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/ListContent.h" #include "eckit/log/JSON.h" #include "eckit/utils/Hash.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec ListContent::classSpec_ = { &Content::classSpec(), "ListContent", }; Reanimator ListContent::reanimator_; ListContent::ListContent() {} ListContent::ListContent(const ValueList& v) { std::copy(v.begin(), v.end(), std::back_inserter(value_)); } ListContent::ListContent(const Value& v) { value_.push_back(v); } ListContent::ListContent(Stream& s) : Content(s) { long count; s >> count; for (int i = 0; i < count; i++) { value_.push_back(Value(s)); } } Content* ListContent::clone() const { ValueList v; v.reserve(value_.size()); for (size_t i = 0; i < value_.size(); ++i) { v.push_back(value_[i].clone()); } return new ListContent(v); } void ListContent::encode(Stream& s) const { Content::encode(s); long count = value_.size(); s << count; for (int i = 0; i < count; ++i) { s << value_[i]; } } ListContent::~ListContent() {} size_t ListContent::size() const { return value_.size(); } void ListContent::value(ValueList& v) const { v = value_; } int ListContent::compare(const Content& other) const { return -other.compareList(*this); } int ListContent::compareList(const ListContent& other) const { if (value_ == other.value_) { return 0; } if (value_ < other.value_) { return -1; } return 1; } void ListContent::json(JSON& s) const { s.startList(); for (size_t i = 0; i < value_.size(); i++) { s << value_[i]; } s.endList(); } void ListContent::print(std::ostream& s) const { s << '('; for (size_t i = 0; i < value_.size(); i++) { if (i > 0) { s << ','; } s << value_[i]; } s << ')'; } Content* ListContent::add(const Content& other) const { return other.addList(*this); } Content* ListContent::addList(const ListContent& other) const { ValueList tmp; std::copy(other.value_.begin(), other.value_.end(), std::back_inserter(tmp)); std::copy(value_.begin(), value_.end(), std::back_inserter(tmp)); return new ListContent(tmp); } void ListContent::append(const Value& value) { value_.push_back(value); } Content* ListContent::sub(const Content& other) const { return other.subList(*this); } Content* ListContent::mul(const Content& other) const { return other.mulList(*this); } Content* ListContent::div(const Content& other) const { return other.divList(*this); } Content* ListContent::mod(const Content& other) const { return other.modList(*this); } void ListContent::value(long long& n) const { if (value_.size() == 1) { n = value_[0]; } else { Content::value(n); } } void ListContent::value(bool& n) const { if (value_.size() == 1) { n = value_[0]; } else { Content::value(n); } } void ListContent::value(double& n) const { if (value_.size() == 1) { n = value_[0]; } else { Content::value(n); } } void ListContent::value(std::string& n) const { if (value_.size() == 1) { n = std::string(value_[0]); } else { Content::value(n); } } void ListContent::value(Date& n) const { if (value_.size() == 1) { n = value_[0]; } else { Content::value(n); } } void ListContent::value(Time& n) const { if (value_.size() == 1) { n = value_[0]; } else { Content::value(n); } } void ListContent::value(DateTime& n) const { if (value_.size() == 1) { n = value_[0]; } else { Content::value(n); } } Value& ListContent::element(const Value& v) { long long n = v; ASSERT(n >= 0 && (size_t)n < value_.size()); return value_.at(n); } bool ListContent::contains(const Value& v) const { long long n = v; return (n >= 0 && (size_t)n < value_.size()); } void ListContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { size_t n = depth; while (n-- > 0) { out << ' '; } } out << '[' << std::endl; for (size_t i = 0; i < value_.size(); i++) { if (i > 0) { out << ',' << std::endl; } value_[i].dump(out, depth + 3); } out << ']'; } void ListContent::hash(Hash& h) const { for (auto v : value_) { v.hash(h); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/MapContent.cc0000664000175000017500000001027415161702250020367 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/value/MapContent.h" #include "eckit/log/JSON.h" #include "eckit/utils/Hash.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec MapContent::classSpec_ = { &Content::classSpec(), "MapContent", }; Reanimator MapContent::reanimator_; MapContent::MapContent() {} MapContent::MapContent(const ValueMap& v) : value_(v) {} MapContent::MapContent(Stream& s) : Content(s) { bool more; s >> more; while (more) { Value k(s); Value v(s); value_[k] = v; s >> more; } } void MapContent::encode(Stream& s) const { Content::encode(s); for (ValueMap::const_iterator j = value_.begin(); j != value_.end(); ++j) { s << true; s << (*j).first; s << (*j).second; } s << false; } MapContent::~MapContent() {} void MapContent::value(ValueMap& v) const { v = value_; } Value MapContent::keys() const { ValueList list; for (ValueMap::const_iterator j = value_.begin(); j != value_.end(); ++j) { list.push_back((*j).first); } return Value::makeList(list); } Value MapContent::remove(const Value& key) { Value result = value_[key]; value_.erase(key); return result; } Value& MapContent::element(const Value& key) { return value_[key]; } bool MapContent::contains(const Value& key) const { return value_.find(key) != value_.end(); } int MapContent::compare(const Content& other) const { return -other.compareMap(*this); } int MapContent::compareMap(const MapContent& other) const { if (value_ == other.value_) { return 0; } if (value_ < other.value_) { return -1; } return 1; } void MapContent::print(std::ostream& s) const { s << '{'; for (ValueMap::const_iterator j = value_.begin(); j != value_.end(); ++j) { if (j != value_.begin()) { s << " , "; } s << (*j).first; s << " => "; s << (*j).second; } s << '}'; } void MapContent::json(JSON& s) const { s.startObject(); for (ValueMap::const_iterator j = value_.begin(); j != value_.end(); ++j) { s << (*j).first; s << (*j).second; } s.endObject(); } Content* MapContent::clone() const { ValueMap v; for (ValueMap::const_iterator j = value_.begin(); j != value_.end(); ++j) { v[(*j).first.clone()] = (*j).second.clone(); } return new MapContent(v); } Content* MapContent::add(const Content& other) const { return other.addMap(*this); } Content* MapContent::sub(const Content& other) const { return other.subMap(*this); } Content* MapContent::mul(const Content& other) const { return other.mulMap(*this); } Content* MapContent::div(const Content& other) const { return other.divMap(*this); } Content* MapContent::mod(const Content& other) const { return other.modMap(*this); } void MapContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { size_t n = depth; while (n-- > 0) { out << ' '; } } out << "{"; const char* sep = "\n"; for (ValueMap::const_iterator j = value_.begin(); j != value_.end(); ++j) { out << sep; (*j).first.dump(out, depth + 3); out << ": "; (*j).second.dump(out, depth + 3, false); sep = ",\n"; } if (!value_.empty()) { out << '\n'; size_t n = depth; while (n-- > 0) { out << ' '; } } out << "}"; } void MapContent::hash(Hash& h) const { for (auto v : value_) { v.first.hash(h); v.second.hash(h); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/DoubleContent.h0000664000175000017500000001027515161702250020727 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Baudouin Raoult /// @author Manuel Fuentes #ifndef eckit_DoubleContent_h #define eckit_DoubleContent_h #include "eckit/value/Content.h" #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class DoubleContent : public Content { protected: // -- Constructor DoubleContent(double); DoubleContent(Stream&); // -- Destructor ~DoubleContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override { Content::value(n); } void value(long long& n) const override; void value(double& n) const override; void value(std::string& n) const override; void value(Date& n) const override { Content::value(n); } void value(Time& n) const override { Content::value(n); } void value(DateTime& n) const override { Content::value(n); } void value(ValueList& n) const override { Content::value(n); } void value(ValueMap& n) const override { Content::value(n); } int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override; int compareDouble(const DoubleContent&) const override; int compareString(const StringContent&) const override { return 1; } int compareNil(const NilContent&) const override { return 1; } int compareList(const ListContent&) const override { return 1; } int compareMap(const MapContent&) const override { return 1; } int compareDate(const DateContent&) const override { return 1; } int compareTime(const TimeContent&) const override { return 1; } int compareDateTime(const DateTimeContent&) const override { return 1; } int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; Content* addDouble(const DoubleContent&) const override; Content* subDouble(const DoubleContent&) const override; Content* mulDouble(const DoubleContent&) const override; Content* divDouble(const DoubleContent&) const override; Value negate() const override; // Content* addNumber(const NumberContent&) const override; // Content* subNumber(const NumberContent&) const override; // Content* mulNumber(const NumberContent&) const override; // Content* divNumber(const NumberContent&) const override; void print(std::ostream&) const override; void json(JSON&) const override; std::string typeName() const override { return "Double"; } bool isDouble() const override { return true; } Content* clone() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: DoubleContent(const DoubleContent&); DoubleContent& operator=(const DoubleContent&); // -- Members double value_; // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; friend class DateContent; friend class NumberContent; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/DateContent.h0000664000175000017500000000726515161702250020377 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Baudouin Raoult /// @author Manuel Fuentes #ifndef eckit_DateContent_h #define eckit_DateContent_h #include "eckit/value/Content.h" #include "eckit/value/Value.h" namespace eckit { class Hash; //---------------------------------------------------------------------------------------------------------------------- class DateContent : public Content { protected: // -- Constructor DateContent(const Date&); DateContent(Stream&); // -- Destructor ~DateContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override { Content::value(n); } void value(long long& n) const override { Content::value(n); } void value(double& n) const override { Content::value(n); } void value(std::string& n) const override { Content::value(n); } void value(Date& n) const override; void value(Time& n) const override { Content::value(n); } void value(DateTime& n) const override { Content::value(n); } void value(ValueList& n) const override { Content::value(n); } void value(ValueMap& n) const override { Content::value(n); } int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override { return -1; } int compareDouble(const DoubleContent&) const override { return -1; } int compareString(const StringContent&) const override { return -1; } int compareNil(const NilContent&) const override { return -1; } int compareList(const ListContent&) const override { return -1; } int compareMap(const MapContent&) const override { return -1; } int compareDate(const DateContent&) const override; int compareTime(const TimeContent&) const override { return 1; } int compareDateTime(const DateTimeContent&) const override { return 1; } int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; Content* subDate(const DateContent&) const override; void print(std::ostream&) const override; void json(JSON&) const override; std::string typeName() const override { return "Date"; } bool isDate() const override { return true; } Content* clone() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: DateContent(const DateContent&); DateContent& operator=(const DateContent&); // -- Members Date value_; // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/StringContent.cc0000664000175000017500000000632015161702250021115 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/JSON.h" #include "eckit/utils/Hash.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Translator.h" #include "eckit/value/StringContent.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec StringContent::classSpec_ = { &Content::classSpec(), "StringContent", }; Reanimator StringContent::reanimator_; StringContent::StringContent(const std::string& s) : value_(s) {} StringContent::StringContent(const char* s) : value_(s) {} StringContent::StringContent(Stream& s) : Content(s) { s >> value_; } Content* StringContent::clone() const { return new StringContent(value_); } void StringContent::encode(Stream& s) const { Content::encode(s); s << value_; } StringContent::~StringContent() {} void StringContent::print(std::ostream& s) const { s << value_; } void StringContent::json(JSON& s) const { s << value_; } int StringContent::compare(const Content& other) const { return -other.compareString(*this); } int StringContent::compareString(const StringContent& other) const { return value_.compare(other.value_); } void StringContent::value(std::string& s) const { s = value_; } void StringContent::value(bool& b) const { std::string v = StringTools::lower(value_); if (v == "true" || v == "on" || v == "yes" || v == "1") { b = true; } else if (v == "false" || v == "off" || v == "no" || v == "0") { b = false; } else { Content::value(b); } } void StringContent::value(long long& l) const { l = Translator()(value_); } void StringContent::value(double& d) const { d = Translator()(value_); } Content* StringContent::add(const Content& other) const { return other.addString(*this); } Content* StringContent::addString(const StringContent& other) const { return new StringContent(other.value_ + value_); } Content* StringContent::sub(const Content& other) const { return other.subString(*this); } Content* StringContent::mul(const Content& other) const { return other.mulString(*this); } Content* StringContent::div(const Content& other) const { return other.divString(*this); } Content* StringContent::mod(const Content& other) const { return other.modString(*this); } void StringContent::dump(std::ostream& out, size_t depth, bool indent) const { if (indent) { while (depth-- > 0) { out << ' '; } } // out << "string(" << value_ << ")"; out << '"' << value_ << '"'; } void StringContent::hash(Hash& h) const { h.add(value_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/value/DispatchParams.h0000664000175000017500000000431215161702250021060 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Florian Rathgeber /// @date March 2015 #ifndef eckit_value_DispatchParams_H #define eckit_value_DispatchParams_H #include "eckit/value/Params.h" //---------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------- template class DispatchParams { public: // methods DispatchParams() {} DispatchParams(Stream& s) { NOTIMP; } static const char* className() { return "eckit::DispatchParams"; } template friend Params::value_t getValue(const DispatchParams& p, const Params::key_t& key); template friend void print(const DispatchParams&, std::ostream&); template friend void encode(const DispatchParams&, Stream&); protected: // members using parametrizer_t = Params::value_t (Derived::*)(const Params::key_t&) const; using store_t = std::map; store_t dispatch_; }; template Params::value_t getValue(const DispatchParams& p, const Params::key_t& key) { typename DispatchParams::store_t::const_iterator i = p.dispatch_.find(key); if (i != p.dispatch_.end()) { typename DispatchParams::parametrizer_t fptr = i->second; const Derived* pobj = static_cast(&p); return (pobj->*fptr)(key); } return Params::value_t(); } template void print(const DispatchParams&, std::ostream&) { NOTIMP; } template void encode(const DispatchParams&, Stream&) { NOTIMP; } //---------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/Properties.h0000664000175000017500000000551215161702250020314 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Properties.h /// @author Tiago Quintino /// @date Jun 2014 #ifndef eckit_Properties_h #define eckit_Properties_h #include #include #include "eckit/value/Params.h" #include "eckit/value/Value.h" namespace eckit { class MD5; //---------------------------------------------------------------------------------------------------------------------- class Properties { public: // types using property_t = Value; using key_t = std::string; public: // methods Properties(); Properties(const property_t&); Properties(Stream&); virtual ~Properties() {} /// @returns true is a property exists bool has(const key_t&) const; /// @returns a property property_t get(const key_t& k) const; /// Sets a property by inserting a new or overwrites an existing property Properties& set(const key_t& k, const property_t& v); /// Sets a property by inserting a new or overwrites an existing property Properties& set(const key_t& k, const Properties& p); /// merge other properties Properties& set(const Properties& p); /// Removes a property bool remove(const key_t& k); /// @returns a property property_t operator[](const key_t& k) const { return get(k); } /// @returns a bool, true if empty false otherwise bool empty() const { return props_.empty(); } static const char* className() { return "eckit::Properties"; } operator property_t() const; void hash(eckit::MD5&) const; protected: void print(std::ostream& s) const; private: // types using PropertyMap = std::map; private: // members PropertyMap props_; //< storage of values protected: // methods void json(JSON& s) const; void encode(Stream& s) const; friend JSON& operator<<(JSON& s, const Properties& v) { v.json(s); return s; } friend std::ostream& operator<<(std::ostream& s, const Properties& v) { v.print(s); return s; } friend Stream& operator<<(Stream& s, const Properties& v) { v.encode(s); return s; } friend property_t getValue(const Properties& p, const key_t& key); friend void print(const Properties& p, std::ostream& s); friend void encode(const Properties& p, Stream& s); friend class Value; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/NilContent.h0000664000175000017500000000733115161702250020236 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Baudouin Raoult /// @author Manuel Fuentes #ifndef eckit_NilContent_h #define eckit_NilContent_h #include "eckit/value/Content.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class NilContent : public Content { protected: // -- Constructors NilContent(); NilContent(Stream&); // -- Destructors ~NilContent() override; // -- Overridden Methods // From Content int compare(const Content& other) const override; void value(bool& n) const override { Content::value(n); } void value(long long& n) const override { Content::value(n); } void value(double& n) const override { Content::value(n); } void value(std::string& n) const override { Content::value(n); } void value(Date& n) const override { Content::value(n); } void value(Time& n) const override { Content::value(n); } void value(DateTime& n) const override { Content::value(n); } void value(ValueList& n) const override; void value(ValueMap& n) const override { Content::value(n); } int compareBool(const BoolContent&) const override { return -1; } int compareNumber(const NumberContent&) const override { return -1; } int compareDouble(const DoubleContent&) const override { return -1; } int compareString(const StringContent&) const override { return -1; } int compareNil(const NilContent&) const override; int compareList(const ListContent&) const override { return 1; } int compareMap(const MapContent&) const override { return 1; } int compareDate(const DateContent&) const override { return 1; } int compareTime(const TimeContent&) const override { return 1; } int compareDateTime(const DateTimeContent&) const override { return 1; } int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; Content* addNil(const NilContent&) const override; Content* subNil(const NilContent&) const override; Content* mulNil(const NilContent&) const override; Content* divNil(const NilContent&) const override; bool isNil() const override { return true; } std::string typeName() const override { return "Nil"; } void print(std::ostream&) const override; void json(JSON&) const override; Content* clone() const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; bool contains(const Value&) const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } void hash(Hash&) const override; private: // -- No copy allowed NilContent(const NilContent&); NilContent& operator=(const NilContent&); // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/value/BoolContent.h0000664000175000017500000001014615161702250020405 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Baudouin Raoult /// @author Manuel Fuentes #ifndef eckit_BoolContent_h #define eckit_BoolContent_h #include "eckit/value/Content.h" #include "eckit/value/Value.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class BoolContent : public Content { protected: // -- Constructor BoolContent(bool); BoolContent(Stream&); // -- Destructor ~BoolContent() override; // -- Overridden methods // -- From Content int compare(const Content& other) const override; void value(bool& n) const override; void value(long long& n) const override; void value(double& n) const override; void value(std::string& n) const override; void value(Date& n) const override { Content::value(n); } void value(Time& n) const override { Content::value(n); } void value(DateTime& n) const override { Content::value(n); } void value(ValueList& n) const override { Content::value(n); } void value(ValueMap& n) const override { Content::value(n); } int compareBool(const BoolContent&) const override; int compareNumber(const NumberContent&) const override { return 1; } int compareDouble(const DoubleContent&) const override { return 1; } int compareString(const StringContent&) const override { return 1; } int compareNil(const NilContent&) const override { return 1; } int compareList(const ListContent&) const override { return 1; } int compareMap(const MapContent&) const override { return 1; } int compareDate(const DateContent&) const override { return 1; } int compareTime(const TimeContent&) const override { return 1; } int compareDateTime(const DateTimeContent&) const override { return 1; } int compareOrderedMap(const OrderedMapContent&) const override { return 1; } Content* add(const Content&) const override; Content* sub(const Content&) const override; Content* mul(const Content&) const override; Content* div(const Content&) const override; Content* mod(const Content&) const override; // Content* addBool(const BoolContent&) const override; // Content* subBool(const BoolContent&) const override; // Content* mulBool(const BoolContent&) const override; // Content* divBool(const BoolContent&) const override; // Content* addNumber(const NumberContent&) const override; // Content* subNumber(const NumberContent&) const override; // Content* mulNumber(const NumberContent&) const override; // Content* divNumber(const NumberContent&) const override; void print(std::ostream&) const override; void dump(std::ostream& out, size_t depth, bool indent = true) const override; void json(JSON&) const override; std::string typeName() const override { return "Bool"; } bool isBool() const override { return true; } Content* clone() const override; void hash(Hash&) const override; // -- From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: BoolContent(const BoolContent&); BoolContent& operator=(const BoolContent&); // -- Members bool value_; // -- Class Members static ClassSpec classSpec_; static Reanimator reanimator_; // -- Friends friend class Reanimator; friend class Value; friend class DateContent; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/option/0000775000175000017500000000000015161702250016200 5ustar alastairalastaireckit-2.0.7/src/eckit/option/EckitTool.h0000664000175000017500000000275015161702250020252 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @author Pedro Maciel /// @date Mar 2016 #pragma once #include "eckit/option/Option.h" #include "eckit/runtime/Tool.h" namespace eckit { namespace option { class Option; class CmdArgs; } // namespace option //---------------------------------------------------------------------------------------------------------------------- class EckitTool : public Tool { protected: // methods EckitTool(int argc, char** argv); public: virtual void usage(const std::string& tool) const = 0; protected: // members std::vector options_; private: // methods virtual void init(const option::CmdArgs&) {} virtual void execute(const option::CmdArgs&) = 0; virtual void finish(const option::CmdArgs&) {} virtual int numberOfPositionalArguments() const { return -1; } virtual int minimumPositionalArguments() const { return -1; } void run() override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/option/Separator.cc0000664000175000017500000000237115161702250020452 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #include "eckit/option/Separator.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/Translator.h" #include namespace eckit::option { Separator::Separator(const std::string& description) : Option("", description) {} Separator::~Separator() {} size_t Separator::set(Configured& parameter, size_t values, args_t::const_iterator begin, args_t::const_iterator end) const { return 0; // Never consumes any argv tokens } void Separator::setDefault(Configured&) const { ; } void Separator::copy(const Configuration& from, Configured& to) const { ; } bool Separator::active() const { return false; } void Separator::print(std::ostream& out) const { out << std::endl << description_ << ":" << std::endl; } } // namespace eckit::option eckit-2.0.7/src/eckit/option/SimpleOption.cc0000664000175000017500000000617715161702250021144 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #pragma once #include #include "eckit/config/Configuration.h" #include "eckit/config/Configured.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/option/SimpleOption.h" #include "eckit/option/Title.h" #include "eckit/utils/Translator.h" namespace eckit::option { template SimpleOption::SimpleOption(const std::string& name, const std::string& description) : base_t(name, description) {} template SimpleOption::SimpleOption(const std::string& name, const std::string& description, const T& default_value) : base_t(name, description, default_value) {} template <> inline size_t SimpleOption::set(Configured& parametrisation, [[maybe_unused]] size_t values, args_t::const_iterator begin, [[maybe_unused]] args_t::const_iterator end) const { // When handling bool, we might not have a value in the range [begin, end), thus the need for this specialization if (values > 0) { // Take first value in range [begin, end) bool value = translate(*begin); set_value(value, parametrisation); return 1; } // Nothing to take from range [begin, end) set_value(true, parametrisation); return 0; } template size_t SimpleOption::set(Configured& parametrisation, [[maybe_unused]] size_t values, args_t::const_iterator begin, args_t::const_iterator end) const { if (begin == end) { throw UserError("No option value found for SimpleOption, where 1 was expected"); } // Take first value in range [begin, end) T value = translate(*begin); set_value(value, parametrisation); return 1; } template void SimpleOption::set_value(const T& value, Configured& parametrisation) const { parametrisation.set(this->name(), value); } template T SimpleOption::translate(const std::string& value) const { T v = Translator()(value); return v; } template <> inline void SimpleOption::copy(const Configuration& from, Configured& to) const { std::string v; if (from.get(name_, v)) { to.set(name_, v); } } template void SimpleOption::copy(const Configuration& from, Configured& to) const { Option::copy(this->name(), from, to); } template <> inline void SimpleOption::print(std::ostream& out) const { out << " --" << name_ << " (" << description_ << ")"; } template void SimpleOption::print(std::ostream& out) const { out << " --" << this->name() << "=" << implementation_detail::Title()() << " (" << this->description() << ")"; } } // namespace eckit::option eckit-2.0.7/src/eckit/option/MultiValueOption.h0000664000175000017500000000376415161702250021643 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/option/Option.h" namespace eckit::option { class MultiValueOption : public BaseOption> { public: using base_t = BaseOption>; using values_t = std::vector; MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values); MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values, size_t n_optional_values); MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values, const values_t& default_values); MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values, size_t n_optional_values, const values_t& default_values); ~MultiValueOption() override = default; size_t set(Configured&, size_t values, args_t::const_iterator begin, args_t::const_iterator end) const override; void print(std::ostream&) const override; private: MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values, size_t n_maximum_values, std::optional default_values); void set_value(const values_t& values, Configured&) const override; [[nodiscard]] values_t translate(const std::string& value) const override; void copy(const Configuration& from, Configured& to) const override; size_t n_mandatory_values_; size_t n_optional_values_; values_t values_; }; } // namespace eckit::option eckit-2.0.7/src/eckit/option/Option.cc0000664000175000017500000000145415161702250017763 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #include "eckit/option/Option.h" #include "eckit/exception/Exceptions.h" namespace eckit::option { Option::Option(const std::string& name, const std::string& description) : name_(name), description_(description) {} std::ostream& operator<<(std::ostream& s, const Option& p) { p.print(s); return s; } } // namespace eckit::option eckit-2.0.7/src/eckit/option/MultiValueOption.cc0000664000175000017500000000747015161702250021777 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/option/MultiValueOption.h" #include #include #include "eckit/config/Configured.h" #include "eckit/exception/Exceptions.h" namespace eckit::option { MultiValueOption::MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values) : MultiValueOption(name, description, n_mandatory_values, 0) {} MultiValueOption::MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values, size_t n_optional_values) : MultiValueOption(name, description, n_mandatory_values, n_optional_values, std::nullopt) {} MultiValueOption::MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values, const values_t& default_values) : MultiValueOption(name, description, n_mandatory_values, 0, default_values) {} MultiValueOption::MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values, size_t n_optional_values, const values_t& default_values) : MultiValueOption(name, description, n_mandatory_values, n_optional_values, std::make_optional(default_values)) {} MultiValueOption::MultiValueOption(const std::string& name, const std::string& description, size_t n_mandatory_values, size_t n_optional_values, std::optional default_values) : base_t(name, description, std::move(default_values)), n_mandatory_values_{n_mandatory_values}, n_optional_values_{n_optional_values} { ASSERT_MSG(n_mandatory_values >= 1, "At least 1 mandatory value is expected."); } size_t MultiValueOption::set(Configured& parametrisation, [[maybe_unused]] size_t values, args_t::const_iterator begin, args_t::const_iterator end) const { if (std::distance(begin, end) < n_mandatory_values_) { throw UserError("Not enough option values found for MultiValueOption, where at least " + std::to_string(n_mandatory_values_) + " were expected"); } // Collect n_mandatory_values_ mandatory values from the range [begin, end) values_t collected; std::copy_n(begin, n_mandatory_values_, std::back_inserter(collected)); begin = begin + n_mandatory_values_; // Collect up to n_optional_values from the range [(updated-)begin, end) // - n.b. collection must stop when either end is reached or when an option (i.e. "--*") is found for (size_t i = 0; i < n_optional_values_; ++i) { if ((begin == end) || (begin->substr(0, 2) == "--")) { break; } collected.push_back(*begin); ++begin; } // Store the collected values set_value(collected, parametrisation); return collected.size(); } void MultiValueOption::set_value(const values_t& values, Configured& parametrisation) const { parametrisation.set(this->name(), values); } MultiValueOption::values_t MultiValueOption::translate(const std::string& value) const { NOTIMP; } void MultiValueOption::copy(const Configuration& from, Configured& to) const { Option::copy(this->name(), from, to); } void MultiValueOption::print(std::ostream& out) const { out << " --" << this->name() << "[=] [" << n_mandatory_values_ << " * values]([" << n_optional_values_ << " * values]) (" << this->description() << ")"; } } // namespace eckit::option eckit-2.0.7/src/eckit/option/FactoryOption.cc0000664000175000017500000000402015161702250021303 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #pragma once #include #include "eckit/config/Configured.h" #include "eckit/exception/Exceptions.h" #include "eckit/option/FactoryOption.h" namespace eckit::option { template FactoryOption::FactoryOption(const std::string& name, const std::string& description) : base_t(name, description) {} template FactoryOption::FactoryOption(const std::string& name, const std::string& description, std::string default_value) : base_t(name, description, std::move(default_value)) {} template size_t FactoryOption::set(Configured& parametrisation, [[maybe_unused]] size_t values, args_t::const_iterator begin, args_t::const_iterator end) const { if (begin == end) { throw UserError("No option value found for FactoryOption, where 1 was expected"); } auto value = translate(*begin); set_value(value, parametrisation); return 1; } template void FactoryOption::set_value(const std::string& value, Configured& parametrisation) const { parametrisation.set(name_, value); } template std::string FactoryOption::translate(const std::string& value) const { return value; } template void FactoryOption::copy(const Configuration& from, Configured& to) const { Option::copy(name_, from, to); } template void FactoryOption::print(std::ostream& out) const { out << " --" << name_ << "=name" << " (" << description_ << ")"; out << std::endl << " Values are: "; T::list(out); } } // namespace eckit::option eckit-2.0.7/src/eckit/option/Option.h0000664000175000017500000000707715161702250017634 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #ifndef Option_H #define Option_H #include #include #include #include #include "eckit/config/Configuration.h" #include "eckit/config/Configured.h" namespace eckit::option { class Option { public: using args_t = std::vector; public: // methods Option(const std::string& name, const std::string& description); Option(const Option&) = delete; Option& operator=(const Option&) = delete; Option(Option&&) = delete; Option& operator=(Option&&) = delete; virtual ~Option() = default; [[nodiscard]] const std::string& name() const { return name_; }; [[nodiscard]] const std::string& description() const { return description_; } [[nodiscard]] virtual bool active() const { return true; }; /** * Set the value of the option into `parameter`, taking as many values as necessary from the range `[begin, end)`. * * - `values` indicates the number of items in `[begin, end)` that were provided as an option value. * This parameter is expected to be either 0 (for --flag options) or 1 (for traditional --option=) * * Return: number of items consumed from the range `[begin, end)`. */ virtual size_t set(Configured& param, size_t values, args_t::const_iterator begin, args_t::const_iterator end) const = 0; virtual void copy(const Configuration& from, Configured& to) const = 0; virtual void setDefault(Configured&) const = 0; template static void copy(const std::string& name, const Configuration& from, Configured& to) { // This is so generic that could probably be provided by Configuration or Configured T value; if (from.get(name, value)) { to.set(name, value); } } protected: // members std::string name_; std::string description_; virtual void print(std::ostream&) const = 0; private: friend std::ostream& operator<<(std::ostream& s, const Option& p); }; template class BaseOption : public Option { public: BaseOption(const std::string& name, const std::string& description) : Option(name, description), default_value_{std::nullopt} {}; BaseOption(const std::string& name, const std::string& description, T default_value) : Option(name, description), default_value_{std::make_optional(std::move(default_value))} {}; BaseOption(const std::string& name, const std::string& description, std::optional default_value) : Option(name, description), default_value_{std::move(default_value)} {}; ~BaseOption() override = default; void setDefault(Configured& parametrisation) const final { if (default_value_) { set_value(default_value_.value(), parametrisation); } } protected: virtual void set_value(const T& value, Configured& parametrisation) const = 0; // Performs the conversion from 'string' value to actual 'type' value virtual T translate(const std::string& value) const = 0; private: std::optional default_value_; }; } // namespace eckit::option #endif eckit-2.0.7/src/eckit/option/Separator.h0000664000175000017500000000357215161702250020320 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #ifndef Separator_H #define Separator_H #include #include "eckit/option/Option.h" namespace eckit::option { class Separator : public Option { public: // -- Exceptions // None // -- Contructors Separator(const std::string& description); // -- Destructor ~Separator() override; // Change to virtual if base class // -- Convertors // None // -- Operators // None // -- Methods size_t set(Configured& parameter, size_t values, args_t::const_iterator begin, args_t::const_iterator end) const override; void setDefault(Configured&) const override; // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // -- Methods void print(std::ostream&) const override; // Change to virtual if base class // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed Separator(const Separator&); Separator& operator=(const Separator&); // -- Members // None // -- Methods // None // -- Overridden methods bool active() const override; void copy(const Configuration& from, Configured& to) const override; // -- Class members // None // -- Class methods // None // -- Friends }; } // namespace eckit::option #endif eckit-2.0.7/src/eckit/option/VectorOption.h0000664000175000017500000000447215161702250021013 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #pragma once #include #include "eckit/option/Option.h" namespace eckit::option { template class VectorOption : public BaseOption> { public: using base_t = BaseOption>; using args_t = Option::args_t; // -- Exceptions // None // -- Contructors VectorOption(const std::string& name, const std::string& description, size_t size, const char* separator = "/"); VectorOption(const std::string& name, const std::string& description, size_t size, const std::vector& default_value, const char* separator = "/"); // -- Destructor ~VectorOption() override = default; // -- Convertors // None // -- Operators // None // -- Methods // None // -- Overridden methods size_t set(Configured&, size_t values, args_t::const_iterator begin, args_t::const_iterator end) const override; // -- Class members // None // -- Class methods // None protected: // -- Members // -- Methods void print(std::ostream&) const override; // Change to virtual if base class // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed VectorOption(const VectorOption&); VectorOption& operator=(const VectorOption&); // -- Members size_t size_; const char* separator_; // -- Methods // None // -- Overridden methods void set_value(const std::vector& value, Configured&) const override; [[nodiscard]] std::vector translate(const std::string& value) const override; void copy(const Configuration& from, Configured& to) const override; // -- Class members // None // -- Class methods // None // -- Friends }; } // namespace eckit::option #include "eckit/option/VectorOption.cc" eckit-2.0.7/src/eckit/option/CmdArgs.cc0000664000175000017500000001423315161702250020032 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Pedro Maciel /// @author Simon Smart /// @date March 2016 #include "eckit/option/CmdArgs.h" #include #include #include "eckit/exception/Exceptions.h" #include "eckit/option/Option.h" #include "eckit/runtime/Main.h" #include "eckit/utils/Tokenizer.h" namespace eckit::option { //---------------------------------------------------------------------------------------------------------------------- CmdArgs::CmdArgs(usage_proc usage, int args_count, int minimum_args, bool throw_on_error) { init(usage, args_count, minimum_args, throw_on_error); } CmdArgs::CmdArgs(usage_proc usage, std::vector& options, int args_count, int minimum_args, bool throw_on_error) { std::swap(options_, options); // Take ownership so it can be destroyed init(usage, args_count, minimum_args, throw_on_error); } CmdArgs::CmdArgs(std::function usage, std::vector& options, int args_count, int minimum_args, bool throw_on_error) { std::swap(options_, options); // Take ownership so it can be destroyed init(usage, args_count, minimum_args, throw_on_error); } void CmdArgs::init(std::function usage, int args_count, int minimum_args, bool throw_on_error) { const Main& ctx = Main::instance(); tool_ = ctx.name(); int argc = ctx.argc(); bool error = false; using options_map_t = std::map; options_map_t opts; // Fill in 'keys_' and prepare options map for (Option* j : options_) { if (j->active()) { ASSERT(opts.find(j->name()) == opts.end()); keys_.insert(j->name()); opts[j->name()] = j; j->setDefault(*this); } } const static std::string prefix = "--"; // Process all options/values in argv, letting each Option collect the necessary entries for (int i = 1; i < argc; ++i) { std::string a = ctx.argv(i); if (a.substr(0, prefix.size()) == prefix) { // An Option 'a' is found (starts with '--')! // The Option might be formatted as --= // ... so we remove the '--' prefix a = a.substr(2); // ... and tokenize [(,)] std::vector tokens = Tokenizer::split_at(a, '='); const std::string name = tokens[0]; tokens.erase(tokens.begin()); if (auto found = opts.find(name); found != opts.end()) { try { const Option* option = found->second; // Given the applicable Option, we prepare the argv tokens (including the ) std::vector remaining; remaining.reserve(tokens.size() + argc); for (const auto& token : tokens) { remaining.push_back(token); } for (int j = i + 1; j < argc; ++j) { remaining.push_back(ctx.argv(j)); } // ... allow the Option to set itself based on all remaining argv tokens size_t consumed = option->set(*this, tokens.size(), std::begin(remaining), std::end(remaining)); // ... and, disregard the number of consumed tokens. i += static_cast(consumed - tokens.size()); } catch (UserError&) { Log::info() << "Invalid value for option --" << name << std::endl; error = true; } } else { Log::info() << "Invalid option --" << name << std::endl; error = true; } } else { // Position argument 'a' is found! args_.push_back(a); } } if (args_count >= 0) { if (args_.size() != size_t(args_count)) { Log::info() << tool_ << ": invalid argument count: expected " << args_count << ", got: " << args_.size() << "." << std::endl; error = true; } } if (minimum_args >= 0) { if (args_.size() < size_t(minimum_args)) { Log::info() << tool_ << ": invalid argument count: expected at least " << minimum_args << ", got: " << args_.size() << std::endl; error = true; } } if (error) { usage(tool_); if (options_.size()) { Log::info() << std::endl; Log::info() << "Options are:" << std::endl; Log::info() << "===========:" << std::endl << std::endl; for (const Option* j : options_) { Log::info() << *j << std::endl << std::endl; } Log::info() << std::endl; } if (throw_on_error) { for (const Option* j : options_) { delete j; } throw UserError("An error occurred in argument parsing", Here()); } ::exit(1); } } CmdArgs::~CmdArgs() { for (const Option* j : options_) { delete j; } } void CmdArgs::configure(Configured& c) const { for (const Option* j : options_) { j->copy(*this, c); } } void CmdArgs::print(std::ostream& out) const { out << "CmdArgs["; LocalConfiguration::print(out); out << "]"; } const std::string& CmdArgs::operator()(size_t i) const { ASSERT(i < args_.size()); return args_[i]; } size_t CmdArgs::count() const { return args_.size(); } const std::string& CmdArgs::tool() const { return tool_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::option eckit-2.0.7/src/eckit/option/EckitTool.cc0000664000175000017500000000243715161702250020412 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "EckitTool.h" #include "eckit/exception/Exceptions.h" #include "eckit/option/CmdArgs.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static EckitTool* INSTANCE = nullptr; static void usage(const std::string& tool) { ASSERT(INSTANCE != nullptr); INSTANCE->usage(tool); } EckitTool::EckitTool(int argc, char** argv) : Tool(argc, argv, "ECKIT_HOME") { ASSERT(INSTANCE == nullptr); INSTANCE = this; } void EckitTool::run() { option::CmdArgs args(&eckit::usage, options_, numberOfPositionalArguments(), minimumPositionalArguments()); init(args); execute(args); finish(args); } void EckitTool::usage(const std::string& tool) const {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/option/FactoryOption.h0000664000175000017500000000325415161702250021155 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #pragma once #include #include "eckit/option/Option.h" namespace eckit::option { /// @note Factory option essentially provides a std::string option, whose acceptable values are listed on the /// command line (typename T may be any class that implements the list() method). It does no checks on /// the validity of input received, and just returns the appropriate string template class FactoryOption : public BaseOption { public: using base_t = BaseOption; FactoryOption(const std::string& name, const std::string& description); FactoryOption(const std::string& name, const std::string& description, std::string default_value); ~FactoryOption() override = default; size_t set(Configured&, size_t values, args_t::const_iterator begin, args_t::const_iterator end) const override; protected: void print(std::ostream&) const override; private: void set_value(const std::string& value, Configured&) const override; [[nodiscard]] std::string translate(const std::string& value) const override; void copy(const Configuration& from, Configured& to) const override; }; } // namespace eckit::option #include "eckit/option/FactoryOption.cc" eckit-2.0.7/src/eckit/option/VectorOption.cc0000664000175000017500000000570515161702250021151 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @author Simon Smart /// @date March 2016 #pragma once #include #include "eckit/exception/Exceptions.h" #include "eckit/option/Title.h" #include "eckit/option/VectorOption.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" namespace eckit::option { template VectorOption::VectorOption(const std::string& name, const std::string& description, size_t size, const char* separator) : base_t(name, description), size_(size), separator_(separator) {} template VectorOption::VectorOption(const std::string& name, const std::string& description, size_t size, const std::vector& default_value, const char* separator) : base_t(name, description, default_value), size_(size), separator_(separator) {} template size_t VectorOption::set(Configured& parametrisation, size_t values, args_t::const_iterator begin, args_t::const_iterator end) const { if (begin == end) { throw UserError("No option value found for VectorOption, where 1 was expected"); } // Take first value in range [begin, end) auto value = translate(*begin); set_value(value, parametrisation); return 1; } template void VectorOption::set_value(const std::vector& value, Configured& parametrisation) const { parametrisation.set(this->name(), value); } template std::vector VectorOption::translate(const std::string& value) const { Translator t; Tokenizer parse(separator_); std::vector tokens; parse(value, tokens); std::vector values; for (size_t i = 0; i < tokens.size(); i++) { values.push_back(t(tokens[i])); } if (size_) { if (values.size() != size_) throw UserError(std::string("Size of supplied vector \"") + this->name() + "\" incorrect", Here()); } return values; } template void VectorOption::print(std::ostream& out) const { out << " --" << this->name(); const char* sep = "="; for (size_t i = 0; i < (size_ ? size_ : 2); i++) { out << sep << implementation_detail::Title()(); sep = separator_; } if (size_ == 0) { out << sep << "..."; } out << " (" << this->description() << ")"; } template void VectorOption::copy(const Configuration& from, Configured& to) const { Option::copy>(this->name(), from, to); } } // namespace eckit::option eckit-2.0.7/src/eckit/option/CmdArgs.h0000664000175000017500000000601015161702250017666 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Pedro Maciel /// @author Simon Smart /// @date March 2016 #ifndef eckit_option_CmdArgs_H #define eckit_option_CmdArgs_H #include #include #include #include "eckit/config/LocalConfiguration.h" namespace eckit::option { //---------------------------------------------------------------------------------------------------------------------- class Option; class CmdArgs : public LocalConfiguration { public: // types using usage_proc = void (*)(const std::string& name); public: // methods CmdArgs(usage_proc usage, int args_count = -1, int minimum_args = 0, bool throw_on_error = false); /// Initialise argument parser with a list of options /// @note Will take ownership of the contents of the vector, and delete them in destructor /// @todo This should probably have some form of smart pointer. CmdArgs(usage_proc usage, std::vector& options, int args_count = -1, int minimum_args = 0, bool throw_on_error = false); CmdArgs(std::function usage, std::vector& options, int args_count = -1, int minimum_args = 0, bool throw_on_error = false); CmdArgs(const CmdArgs&) = delete; CmdArgs& operator=(const CmdArgs&) = delete; CmdArgs(CmdArgs&&) = delete; CmdArgs& operator=(CmdArgs&&) = delete; ~CmdArgs(); // Accessors // const std::set& keys() const; // const std::vector& args() const; // const std::string& args(size_t) const; const std::string& operator()(size_t) const; const std::string& tool() const; size_t count() const; void configure(Configured&) const; // has, get and set methods are inherited from LocalConfiguration in their entirety std::vector::const_iterator begin() const { return args_.begin(); } std::vector::const_iterator end() const { return args_.end(); } std::vector::iterator begin() { return args_.begin(); } std::vector::iterator end() { return args_.end(); } private: // methods void init(std::function usage, int args_count, int minumum_args, bool throw_on_errror); void print(std::ostream&) const override; private: // members std::set keys_; std::vector args_; std::vector options_; std::string tool_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::option #endif // eckit_option_CmdArgs_H eckit-2.0.7/src/eckit/option/Title.h0000664000175000017500000000254415161702250017437 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_option_Title_H #define eckit_option_Title_H #include #include "eckit/filesystem/PathName.h" namespace eckit::option { namespace implementation_detail { /* * The following is a utility used to automatically determine the description of an option type. */ template struct Title { const char* operator()() const; }; template <> inline const char* Title::operator()() const { return "ordinal"; } template <> inline const char* Title::operator()() const { return "integer"; } template <> inline const char* Title::operator()() const { return "real"; } template <> inline const char* Title::operator()() const { return "0/1"; } template <> inline const char* Title::operator()() const { return "string"; } template <> inline const char* Title::operator()() const { return "path"; } } // namespace implementation_detail } // namespace eckit::option #endif eckit-2.0.7/src/eckit/option/CMakeLists.txt0000664000175000017500000000117315161702250020742 0ustar alastairalastairecbuild_add_library( TARGET eckit_option TYPE SHARED INSTALL_HEADERS LISTED HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/option SOURCES FactoryOption.cc FactoryOption.h MultiValueOption.cc MultiValueOption.h Option.cc Option.h Separator.cc Separator.h SimpleOption.cc SimpleOption.h Title.h VectorOption.cc VectorOption.h CmdArgs.cc CmdArgs.h EckitTool.cc EckitTool.h TEMPLATES FactoryOption.cc VectorOption.cc SimpleOption.cc PUBLIC_LIBS eckit ) eckit-2.0.7/src/eckit/option/SimpleOption.h0000664000175000017500000000256115161702250020777 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Apr 2015 #pragma once #include #include "eckit/option/Option.h" namespace eckit::option { template class SimpleOption : public BaseOption { public: using base_t = BaseOption; using args_t = Option::args_t; SimpleOption(const std::string& name, const std::string& description); SimpleOption(const std::string& name, const std::string& description, const T& default_value); ~SimpleOption() override = default; size_t set(Configured&, size_t values, args_t::const_iterator begin, args_t::const_iterator end) const override; protected: void print(std::ostream&) const override; private: void set_value(const T& value, Configured&) const override; [[nodiscard]] T translate(const std::string& value) const override; void copy(const Configuration& from, Configured& to) const override; }; } // namespace eckit::option #include "eckit/option/SimpleOption.cc" eckit-2.0.7/src/eckit/container/0000775000175000017500000000000015161702250016652 5ustar alastairalastaireckit-2.0.7/src/eckit/container/KDTree.h0000664000175000017500000000547415161702250020153 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef KDTree_H #define KDTree_H #include "eckit/container/kdtree/KDNode.h" #include "eckit/container/sptree/SPTree.h" #include "KDMapped.h" #include "KDMemory.h" namespace eckit { template class KDTreeX : public SPTree> { public: using Node = KDNode; using SPTreeType = SPTree; // cannot redefine as SPTree since some compilers in-class redefinitions using Alloc = typename Traits::Alloc; using Value = typename SPTreeType::Value; using Point = typename SPTreeType::Point; using Payload = typename SPTreeType::Payload; public: KDTreeX(Alloc& alloc) : SPTreeType(alloc) {} /// ITER must be a random access iterator /// WARNING: container is changed (sorted) template void build(ITER begin, ITER end) { Alloc& a = this->alloc_; this->root_ = a.convert(Node::build(a, begin, end)); a.root(this->root_); } /// Container must be a random access /// WARNING: container is changed (sorted) template void build(Container& c) { typename Container::iterator b = c.begin(); typename Container::iterator e = c.end(); build(b, e); } // void insert(const Value& value) { Alloc& a = this->alloc_; Node* root = Node::insert(a, value, a.convert(this->root_, (Node*)0), 0); if (a.convert(root) != this->root_) { a.root(a.convert(root)); this->root_ = a.convert(root); } } }; template class KDTreeMemory : public KDTreeX> { KDMemory alloc_; public: using KDTree = KDTreeX>; using Value = typename KDTree::Value; using Point = typename KDTree::Point; using Payload = typename KDTree::Payload; public: KDTreeMemory() : KDTree(alloc_) {} }; template class KDTreeMapped : public KDTreeX> { KDMapped alloc_; public: using KDTree = KDTreeX>; using Value = typename KDTree::Value; using Point = typename KDTree::Point; using Payload = typename KDTree::Payload; using Node = typename KDTree::Node; public: KDTreeMapped(const eckit::PathName& path, size_t itemCount, size_t metadataSize) : KDTree(alloc_), alloc_(path, itemCount, sizeof(Node), metadataSize) {} }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/bsptree/0000775000175000017500000000000015161702250020316 5ustar alastairalastaireckit-2.0.7/src/eckit/container/bsptree/BSPHyperPlane.h0000664000175000017500000000165615161702250023113 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef BSPHyperPlane_H #define BSPHyperPlane_H namespace eckit { template class BSPHyperPlane { Point normal_; double d_; public: BSPHyperPlane() : normal_(), d_() {} BSPHyperPlane(const Point& normal, const Point& point) : normal_(Point::normalize(normal)), d_(-Point::dot(normal_, point)) {} double position(const Point& p) const { return Point::dot(p, normal_) + d_; } const Point& normal() const { return normal_; } double d() const { return d_; } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/bsptree/BSPNode.cc0000664000175000017500000001443715161702250022070 0ustar alastairalastair/* * (C) Copythis->right 1996-2013 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef BSPNode_CC #define BSPNode_CC #include #include "eckit/eckit.h" #include #include #include #include #include #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" namespace eckit { // The hyperplane is define by the vector (l, r) passing through the middle point template BSPNode::BSPNode(const Value& v, const HyperPlane& plane, double dist) : SPNodeType(v), plane_(plane), dist_(dist) {} template void BSPNode::nearestNeighbourX(Alloc& a, const Point& p, Node*& best, double& max, int depth) { a.statsVisitNode(); if (this->left_ && this->right_) { // Check in which half the point lies double d = plane_.position(p); // See if we need to visit both double distanceToPlane = fabs(d); // distanceToPlane = 0; if (d <= 0) { this->left(a)->nearestNeighbourX(a, p, best, max, depth + 1); double dd = this->right(a)->dist_; if (distanceToPlane + dd <= max) { a.statsCrossOver(); this->right(a)->nearestNeighbourX(a, p, best, max, depth + 1); } } else { this->right(a)->nearestNeighbourX(a, p, best, max, depth + 1); double dd = this->left(a)->dist_; if (distanceToPlane + dd <= max) { a.statsCrossOver(); this->left(a)->nearestNeighbourX(a, p, best, max, depth + 1); } } } else { if (this->left_) { this->left(a)->nearestNeighbourX(a, p, best, max, depth + 1); return; } if (this->right_) { this->right(a)->nearestNeighbourX(a, p, best, max, depth + 1); return; } ASSERT(!this->left_ || !this->right_); double d = Point::distance(p, this->value_.point()); if (d < max) { max = d; best = this; a.statsNewCandidateOK(); } else { a.statsNewCandidateMiss(); } } } //---------------------------------------------------------------------------------------------------------------------- template void BSPNode::kNearestNeighboursX(Alloc& a, const Point& p, size_t k, NodeQueue& result, int depth) { if (this->left_ && this->right_) { // Check in which half the point lies double d = plane_.position(p); // See if we need to visit both double distanceToPlane = fabs(d); double max = result.largest(); if (d <= 0) { this->left(a)->kNearestNeighboursX(a, p, k, result, depth + 1); double dd = this->right(a)->dist_; if (result.incomplete() || distanceToPlane + dd <= max) { a.statsCrossOver(); this->right(a)->kNearestNeighboursX(a, p, k, result, depth + 1); } } else { this->right(a)->kNearestNeighboursX(a, p, k, result, depth + 1); double dd = this->left(a)->dist_; if (result.incomplete() || distanceToPlane + dd <= max) { a.statsCrossOver(); this->left(a)->kNearestNeighboursX(a, p, k, result, depth + 1); } } return; } if (this->left_) { this->left(a)->kNearestNeighboursX(a, p, k, result, depth + 1); return; } if (this->right_) { this->right(a)->kNearestNeighboursX(a, p, k, result, depth + 1); return; } // This is a leaf double d = Point::distance(p, this->value_.point()); result.push(this, a.convert(this), d); } template template double BSPNode::distanceToPlane(const Container& in, const HyperPlane& plane) { double min = std::numeric_limits::max(); for (typename Container::const_iterator j = in.begin(); j != in.end(); ++j) { const Point& p = (*j).point(); // Find the closest value to the partitionning plan double dist = fabs(plane.position(p)); if (dist < min) { min = dist; } } return min; } template template BSPNode* BSPNode::build(Alloc& a, Partition& p, const Container& nodes, double dist, int depth) { HyperPlane plane; if (nodes.size() == 0) return 0; a.statsDepth(depth); if (nodes.size() == 1) { return a.newNode3(nodes[0], plane, dist, (BSPNode*)0); } Container left; Container right; p(nodes, left, right, plane, depth); if (left.size() == 0 || right.size() == 0) { ASSERT(left.size() == 1 || right.size() == 1); if (left.size() == 1) { return a.newNode3(left[0], plane, dist, (BSPNode*)0); } else { return a.newNode3(right[0], plane, dist, (BSPNode*)0); } } ASSERT(left.size() < nodes.size()); ASSERT(right.size() < nodes.size()); ASSERT(right.size() + left.size() == nodes.size()); BSPNode* n = a.newNode3(nodes[0], plane, dist, (BSPNode*)0); double dl = distanceToPlane(left, plane); double dr = distanceToPlane(right, plane); // if(depth == 1) { // std::cerr << Partition::name() << " distanceToPlane " << dl << " " << dr << std::endl; //} n->left(a, build(a, p, left, dl, depth + 1)); n->right(a, build(a, p, right, dr, depth + 1)); return n; } template void BSPNode::findInSphereX(Alloc& a, const Point& p, double radius, NodeList& result, int depth) { NOTIMP; } } // namespace eckit #endif eckit-2.0.7/src/eckit/container/bsptree/BSPNode.h0000664000175000017500000000366115161702250021727 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef BSPNode_H #define BSPNode_H #include "BSPHyperPlane.h" #include "eckit/container/sptree/SPNode.h" namespace eckit { template class BSPNode : public SPNode> { public: using SPNodeType = SPNode>; using Value = typename SPNodeType::Value; using Alloc = typename SPNodeType::Alloc; using Point = typename SPNodeType::Point; using NodeList = typename SPNodeType::NodeList; using NodeQueue = typename SPNodeType::NodeQueue; using NodeInfo = typename SPNodeType::NodeInfo; using HyperPlane = BSPHyperPlane; using Node = BSPNode; private: HyperPlane plane_; double dist_; // Distance to parent's hyperplane public: BSPNode(const Value& v, const HyperPlane& plane, double dist); ~BSPNode() override {} template static BSPNode* build(Alloc& a, Partition& p, const Container& nodes, double dist, int depth = 0); private: virtual void nearestNeighbourX(Alloc& a, const Point& p, Node*& best, double& max, int depth); virtual void findInSphereX(Alloc& a, const Point& p, double radius, NodeList& result, int depth); virtual void kNearestNeighboursX(Alloc& a, const Point& p, size_t k, NodeQueue& result, int depth); //========================== template static double distanceToPlane(const Container& in, const HyperPlane& plane); }; } // namespace eckit #include "BSPNode.cc" #endif eckit-2.0.7/src/eckit/container/Trie.cc0000664000175000017500000000752215161702250020072 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/container/Trie.h" #include "eckit/exception/Exceptions.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template Trie::Trie() : from_(0), set_(0), kids_(0) {} template Trie::~Trie() { for (size_t i = 0; i < kids_.size(); i++) { if (kids_[i]) delete (kids_[i]); } } template void Trie::insert(const std::string& key, T value) { const char* k = key.c_str(); Trie* x = find((unsigned char*)k, true); x->value_ = value; x->set_ = true; } template void Trie::remove(const std::string& key) { const char* k = key.c_str(); remove((unsigned char*)k); } template T* Trie::find(const std::string& key) const { const char* k = key.c_str(); Trie* x = const_cast*>(this)->find((unsigned char*)k, false); return (x && x->set_) ? &x->value_ : 0; } template bool Trie::contains(const std::string& key) const { return find(key) != 0; } template bool Trie::remove(const unsigned char* key) { if (*key == 0) { set_ = 0; return (kids_.size() == 0); // Remove me if size if 0 } int pos = int(*key) - int(from_); if (pos >= 0 && pos < int(kids_.size()) && kids_[pos]) { if (kids_[pos]->remove(key + 1)) { delete kids_[pos]; kids_[pos] = 0; // May be we should shrink the list here for (size_t i = 0; i < kids_.size(); i++) if (kids_[i] != 0) return false; // Delete me if I am not also a value kids_.clear(); return !set_; } } return false; } template Trie* Trie::find(const unsigned char* key, bool make) { if (*key == 0) { return this; } int pos = int(*key) - int(from_); if (pos >= 0 && pos < int(kids_.size()) && kids_[pos]) { return kids_[pos]->find(key + 1, make); } if (make) { // If outside of range, we need to expand the storage vector. if (pos < 0 || pos >= int(kids_.size())) { int from = 0; size_t sz; if (kids_.empty()) // First in { sz = 1; from_ = *key; } else if (pos < 0) // Insert before { sz = kids_.size() - pos; from_ = *key; from = -pos; } else // Insert after { sz = pos + 1; } std::vector*> kids(sz, 0); std::copy(kids_.begin(), kids_.end(), kids.begin() + from); kids_ = kids; pos = int(*key) - int(from_); } ASSERT(kids_[pos] == 0); Trie* k = new Trie; kids_[pos] = k; return k->find(key + 1, make); } return 0; } template void Trie::print(std::ostream& s) const { if (set_) s << "(" << value_ << ")"; for (size_t i = 0; i < kids_.size(); i++) { if (kids_[i]) { s << (unsigned char)(from_ + i) << ' '; kids_[i]->print(s); } } if (kids_.empty()) s << std::endl; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/container/CacheManager.h0000664000175000017500000002502515161702250021325 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Baudouin Raoult /// @date May 2015 #ifndef eckit_container_CacheManager_h #define eckit_container_CacheManager_h #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathExpander.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/FileLock.h" #include "eckit/log/Log.h" #include "eckit/os/AutoUmask.h" #include "eckit/os/Semaphore.h" #include "eckit/thread/AutoLock.h" #include "eckit/types/FixedString.h" #include "eckit/utils/MD5.h" #include "eckit/utils/Tokenizer.h" namespace eckit { template class BTree; class BTreeLock; //---------------------------------------------------------------------------------------------------------------------- /// Filesystem Cache Manager class CacheManagerBase { public: CacheManagerBase(const std::string& loaderName, size_t maxCacheSize, const std::string& extension); CacheManagerBase(const CacheManagerBase&) = delete; CacheManagerBase(CacheManagerBase&&) = delete; CacheManagerBase& operator=(const CacheManagerBase&) = delete; CacheManagerBase& operator=(CacheManagerBase&&) = delete; ~CacheManagerBase(); std::string loader() const; protected: void touch(const PathName& base, const PathName& path) const; void rescanCache(const eckit::PathName& base) const; bool writable(const PathName& path) const; private: std::string loaderName_; size_t maxCacheSize_; std::string extension_; using cache_key_t = FixedString; struct cache_entry_t { size_t size_; size_t count_; time_t last_; }; using cache_btree_t = BTree; mutable std::unique_ptr btree_; }; //---------------------------------------------------------------------------------------------------------------------- class CacheManagerNoLock { public: CacheManagerNoLock() = default; void lock() {} void unlock() {} }; //---------------------------------------------------------------------------------------------------------------------- class CacheManagerFileSemaphoreLock { PathName path_; eckit::Semaphore lock_; public: explicit CacheManagerFileSemaphoreLock(const std::string& path); void lock(); void unlock(); }; //---------------------------------------------------------------------------------------------------------------------- class CacheManagerFileFlock { eckit::FileLock lock_; public: explicit CacheManagerFileFlock(const std::string& path); void lock(); void unlock(); }; //---------------------------------------------------------------------------------------------------------------------- template class CacheManager : public CacheManagerBase { public: // methods using value_type = typename Traits::value_type; class CacheContentCreator { public: virtual ~CacheContentCreator() = default; virtual void create(const PathName&, value_type& value, bool& saved) = 0; }; using key_t = std::string; public: // methods explicit CacheManager(const std::string& loaderName, const std::string& roots, bool throwOnCacheMiss, size_t maxCacheSize); PathName getOrCreate(const key_t& key, CacheContentCreator& creator, value_type& value) const; private: // methods bool get(const key_t& key, PathName& path) const; PathName stage(const key_t& key, const PathName& root) const; bool commit(const key_t& key, const PathName& path, const PathName& root) const; PathName entry(const key_t& key, const std::string& root) const; PathName base(const std::string& root) const; private: // members std::vector roots_; bool throwOnCacheMiss_; }; //---------------------------------------------------------------------------------------------------------------------- // NOTE : this should be in the .cc but we have the non-template CacheManagerBase there not to have duplicate symbols template CacheManager::CacheManager(const std::string& loaderName, const std::string& roots, bool throwOnCacheMiss, size_t maxCacheSize) : CacheManagerBase(loaderName, maxCacheSize, Traits::extension()), throwOnCacheMiss_(throwOnCacheMiss) { eckit::Tokenizer parse(":"); std::vector v; parse(roots, v); for (auto& root : v) { // entries with e.g. {CWDFS}/cache will be expanded with PathExpander factory CWDFS PathName p = PathExpander::expand(root); if (not p.exists()) { Log::warning() << "CACHE-MANAGER " << Traits::name() << ", " << p << " does not exist" << std::endl; try { if (writable(p.dirName())) { AutoUmask umask(0); p.mkdir(0777); Log::warning() << "CACHE-MANAGER " << Traits::name() << ", " << p << " created" << std::endl; } else { Log::debug() << "CACHE-MANAGER " << Traits::name() << ", " << p.dirName() << " not writable" << std::endl; } } catch (FailedSystemCall&) { // ignore } } if (p.exists()) { // test again, we may have just created it roots_.push_back(p); } } Log::debug() << "CACHE-MANAGER " << Traits::name() << ", roots defined and found or created: " << roots_ << std::endl; } template bool CacheManager::get(const key_t& key, PathName& path) const { for (const auto& root : roots_) { auto p = entry(key, root); if (p.exists()) { path = p; Log::debug() << "CACHE-MANAGER found path " << p << std::endl; touch(base(root), p); return true; } } if (throwOnCacheMiss_) { std::ostringstream oss; oss << "CacheManager cache miss: key=" << key << ", tried:"; const char* sep = " "; for (const auto& root : roots_) { oss << sep << entry(key, root); sep = ", "; } throw UserError(oss.str()); } return false; } template PathName CacheManager::base(const std::string& root) const { std::ostringstream oss; oss << root << "/" << Traits::name(); return oss.str(); } template PathName CacheManager::entry(const key_t& key, const std::string& root) const { std::ostringstream oss; oss << base(root).asString() << "/" << Traits::version() << "/" << key << Traits::extension(); return oss.str(); } template PathName CacheManager::stage(const key_t& key, const PathName& root) const { auto p = entry(key, root); AutoUmask umask(0); // FIXME: mask does not seem to affect first level directory p.dirName().mkdir(0777); // ensure directory exists Log::info() << "CacheManager creating file " << p << std::endl; // unique file name avoids race conditions on the file from multiple processes return PathName::unique(p); } template bool CacheManager::commit(const key_t& key, const PathName& path, const PathName& root) const { // path is the temporary file auto file = entry(key, root); try { SYSCALL(::chmod(path.asString().c_str(), 0444)); PathName::rename(path, file); } catch (FailedSystemCall& e) { // ignore failed system call -- another process nay have created the file meanwhile Log::debug() << "Failed rename of cache file -- " << e.what() << std::endl; return false; } return true; } template PathName CacheManager::getOrCreate(const key_t& key, CacheContentCreator& creator, value_type& value) const { PathName path; if (get(key, path)) { Log::debug() << "Loading cache file " << path << std::endl; Traits::load(*this, value, path); return path; } for (const PathName& root : roots_) { if (not writable(root)) { Log::debug() << "CACHE-MANAGER root " << root << " isn't writable, skipping... " << std::endl; continue; } auto file = entry(key, root); Log::info() << "Cache file " << file << " does not exist" << std::endl; try { typename Traits::Locker locker(file); AutoLock lock(locker); if (!get(key, path)) { Log::info() << "Creating cache file " << file << std::endl; PathName tmp = stage(key, root); bool saved = false; // The creator may decide to save creator.create(tmp, value, saved); if (!saved) { Traits::save(*this, value, tmp); } ASSERT(commit(key, tmp, root)); ASSERT(get(key, path)); // this includes the call to touch(path) // We reload from cache so we use the proper loader, e.g. mmap of shared-mem... Traits::load(*this, value, path); } else { Log::debug() << "Loading cache file " << file << " (created by another process)" << std::endl; // touch() is done in the (successful) get() above Traits::load(*this, value, path); } return path; } catch (FailedSystemCall& e) { Log::error() << "Error creating cache file: " << file << " (" << e.what() << ")" << std::endl; } } std::ostringstream oss; oss << "CacheManager cannot create key=" << key << ", tried:"; const char* sep = " "; for (const PathName& root : roots_) { PathName p = entry(key, root); oss << sep << p; sep = ", "; } throw UserError(oss.str()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/container/BloomFilter.cc0000664000175000017500000000450515161702250021403 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/container/BloomFilter.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/MD5.h" #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template BloomFilter::BloomFilter(size_t size) : size_(size), entries_(0), data_(elementCount(size), 0) {} template BloomFilter::~BloomFilter() {} template bool BloomFilter::empty() const { return (entries_ == 0); } template void BloomFilter::insert(const T& value) { size_t bit_index = index(value); size_t elem = bit_index / sizeof(data_type); size_t offset = bit_index % sizeof(data_type); data_[elem] |= (data_type(1) << offset); entries_++; } template bool BloomFilter::contains(const T& value) const { size_t bit_index = index(value); size_t elem = bit_index / sizeof(data_type); size_t offset = bit_index % sizeof(data_type); return !((data_[elem] & (data_type(1) << offset)) == 0); } template void BloomFilter::print(std::ostream& s) const { s << "BloomFilter(size=" << size_ << ",entries=" << entries_ << ")"; } template size_t BloomFilter::elementCount(size_t nbits) { return (nbits == 0) ? 0 : ((nbits - 1) / sizeof(data_type)) + 1; } template size_t BloomFilter::index(const T& value) const { // n.b. We could use any other hash, but MD5 is already in here. MD5 md5; unsigned char md5_buf[16]; md5.add(value); md5.numericalDigest(md5_buf); // Take the modules incrementally size_t idx = 0; for (size_t i = 0; i < sizeof(md5_buf); i++) idx = ((idx * 256) + md5_buf[i]) % size_; return idx; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/container/sptree/0000775000175000017500000000000015161702250020154 5ustar alastairalastaireckit-2.0.7/src/eckit/container/sptree/SPNodeInfo.h0000664000175000017500000000336615161702250022301 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SPNodeInfo_H #define SPNodeInfo_H #include #include namespace eckit { template class SPNode; template class SPTreeIterator; template class SPValue; template struct SPNodeInfo { using Point = typename Traits::Point; using Payload = typename Traits::Payload; using Alloc = typename Traits::Alloc; using Value = SPValue; using ID = typename Alloc::Ptr; using Node = NodeType; const Node* node_; ID id_; double distance_; public: SPNodeInfo() : node_(0), id_(0), distance_(0) {} SPNodeInfo(const Node* node, ID id, double distance) : node_(node), id_(id), distance_(distance) {} ID id() const { return id_; } bool operator<(const SPNodeInfo& other) const { return distance_ < other.distance_; } using NodeList = std::vector; const Point& point() const { return node_->point(); } const Payload& payload() const { return node_->payload(); } const Value& value() const { return node_->value(); } double distance() const { return distance_; } friend std::ostream& operator<<(std::ostream& s, const SPNodeInfo& p) { s << "[value=" << p.value() << ",distance=" << p.distance() << "]"; return s; } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/sptree/SPValue.h0000664000175000017500000000276015161702250021651 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SPValue_H #define SPValue_H #include namespace eckit { template class SPValue { public: using Point = typename Traits::Point; using Payload = typename Traits::Payload; Point point_; Payload payload_; public: SPValue(const Point& point, const Payload& payload) : point_(point), payload_(payload) {} template SPValue(const V& v) : point_(v.point()), payload_(v.payload()) {} const Point& point() const { return point_; } const Payload& payload() const { return payload_; } Point& point() { return point_; } // FIXME: remove this one Payload& payload() { return payload_; } void point(const Point& p) const { point_ = p; } void payload(const Payload& p) const { payload_ = p; } void print(std::ostream& o) const { o << "(point=" << point_ << ",payload=" << payload_ << ")"; } friend std::ostream& operator<<(std::ostream& o, const SPValue& t) { t.print(o); return o; } bool operator<(const SPValue& other) const { return point() < other.point(); } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/sptree/SPIterator.h0000664000175000017500000000326315161702250022365 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SPIterator_H #define SPIterator_H #include "eckit/container/sptree/SPValue.h" namespace eckit { template class SPIterator { using Point = typename Traits::Point; using Payload = typename Traits::Payload; using Alloc = typename Traits::Alloc; using Value = SPValue; using Ptr = typename Alloc::Ptr; using ID = typename Alloc::Ptr; using Node = NodeType; Alloc& alloc_; Ptr ptr_; public: SPIterator(Alloc& alloc, Ptr ptr) : alloc_(alloc), ptr_(ptr) { // std::cout << "SPIterator " << ptr << std::endl; Node* node = alloc_.convert(ptr_, (Node*)0); if (node) { if (!node->next(alloc_)) { Node* prev = 0; node->linkNodes(alloc, prev); } } } bool operator!=(const SPIterator& other) { return ptr_ != other.ptr_; } operator Value*() { Node* n = alloc_.convert(ptr_, (Node*)0); return &(n->value()); } Value* operator->() { Node* n = alloc_.convert(ptr_, (Node*)0); return &(n->value()); } SPIterator& operator++() { ptr_ = alloc_.convert(ptr_, (Node*)0)->next_; return *this; } ID nodeID() const { return ptr_; } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/sptree/SPMetadata.h0000664000175000017500000000115115161702250022306 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SPMetadata_H #define SPMetadata_H namespace eckit { template struct SPMetadata { using Point = typename Traits::Point; Point offset_; Point scale_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/sptree/SPTree.h0000664000175000017500000001063715161702250021476 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SPTree_H #define SPTree_H #include "eckit/container/sptree/SPIterator.h" #include "eckit/container/sptree/SPMetadata.h" #include "eckit/container/sptree/SPNode.h" #include "eckit/exception/Exceptions.h" namespace eckit { template class SPTree { public: using Point = typename Traits::Point; using Payload = typename Traits::Payload; using Alloc = typename Traits::Alloc; using Ptr = typename Alloc::Ptr; using ID = typename Alloc::Ptr; using Node = NodeType; using Metadata = SPMetadata; using PointType = Point; using PayloadType = Payload; using NodeList = typename Node::NodeList; using NodeInfo = SPNodeInfo; using Value = typename Node::Value; Alloc& alloc_; Ptr root_; Metadata meta_; using iterator = SPIterator; using value_type = std::pair; public: SPTree(Alloc& alloc) : alloc_(alloc), root_(0) {} ~SPTree() { alloc_.deleteNode(root_, (Node*)0); } void setMetadata(const Point& offset, const Point& scale) { meta_.offset_ = offset; meta_.scale_ = scale; alloc_.setMetadata(meta_); } NodeInfo nodeByID(ID id) { return SPNodeInfo(alloc_.convert(id, (Node*)0), id, 0.0); } void getMetadata(Point& offset, Point& scale) { alloc_.getMetadata(meta_); offset = meta_.offset_; scale = meta_.scale_; } NodeInfo nearestNeighbour(const Point& p) { if (!root_) { root_ = alloc_.root(); } alloc_.statsCall(); ASSERT(root_); return alloc_.convert(root_, (Node*)0)->nearestNeighbour(alloc_, p); } NodeList findInSphere(const Point& p, double radius) { if (!root_) { root_ = alloc_.root(); } alloc_.statsCall(); ASSERT(root_); return alloc_.convert(root_, (Node*)0)->findInSphere(alloc_, p, radius); } NodeList kNearestNeighbours(const Point& p, size_t k) { if (!root_) { root_ = alloc_.root(); } alloc_.statsCall(); ASSERT(root_); return alloc_.convert(root_, (Node*)0)->kNearestNeighbours(alloc_, p, k); } // For testing only... NodeInfo nearestNeighbourBruteForce(const Point& p) { if (!root_) { root_ = alloc_.root(); } alloc_.statsCall(); ASSERT(root_); return alloc_.convert(root_, (Node*)0)->nearestNeighbourBruteForce(alloc_, p); } NodeList findInSphereBruteForce(const Point& p, double radius) { if (!root_) { root_ = alloc_.root(); } ASSERT(root_); return alloc_.convert(root_, (Node*)0)->findInSphereBruteForce(alloc_, p, radius); } NodeList kNearestNeighboursBruteForce(const Point& p, size_t k) { if (!root_) { root_ = alloc_.root(); } ASSERT(root_); return alloc_.convert(root_, (Node*)0)->kNearestNeighboursBruteForce(alloc_, p, k); } template void visit(Visitor& v) { if (!root_) { root_ = alloc_.root(); } ASSERT(root_); return alloc_.convert(root_, (Node*)0)->visit(alloc_, v); } void statsReset() { alloc_.statsReset(); } void statsPrint(std::ostream& o, bool fancy) const { if (fancy) { o << *this << ": "; } alloc_.statsPrint(o, fancy); } void print(std::ostream& o) const { o << "SPTree"; } friend std::ostream& operator<<(std::ostream& o, const SPTree& t) { t.print(o); return o; } iterator begin() { if (empty()) { return end(); } if (!root_) { root_ = alloc_.root(); } ASSERT(root_); return iterator(alloc_, root_); } iterator end() { return iterator(alloc_, 0); } bool empty() const { return size() == 0; } size_t size() const { return alloc_.nbItems(); } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/sptree/SPNode.cc0000664000175000017500000001317315161702250021620 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SPNode_CC #define SPNode_CC #include #include #include #include #include #include #include namespace eckit { template template SPNode::SPNode(const V& value) : value_(value), left_(0), right_(0), next_(0) {} template SPNodeInfo SPNode::nearestNeighbour(Alloc& a, const Point& p) { double max = Point::distance(p, value_.point()); Node* best = this->asNode(); asNode()->nearestNeighbourX(a, p, best, max, 0); return NodeInfo(best, a.convert(best), max); } template SPNodeInfo SPNode::nearestNeighbourBruteForce(Alloc& a, const Point& p) { double max = Point::distance(p, value_.point()); Node* best = this->asNode(); asNode()->nearestNeighbourBruteForceX(a, p, best, max, 0); return NodeInfo(best, a.convert(best), max); } template void SPNode::nearestNeighbourBruteForceX(Alloc& a, const Point& p, Node*& best, double& max, int depth) { double d = Point::distance(p, value_.point()); if (d < max) { best = this->asNode(); max = d; } if (left_) left(a)->nearestNeighbourBruteForceX(a, p, best, max, depth + 1); if (right_) right(a)->nearestNeighbourBruteForceX(a, p, best, max, depth + 1); } //---------------------------------------------------------------------------------------------------------------------- template typename SPNode::NodeList SPNode::kNearestNeighbours(Alloc& a, const Point& p, size_t k) { NodeQueue queue(k); NodeList result; asNode()->kNearestNeighboursX(a, p, k, queue, 0); queue.fill(result); return result; } template void SPNode::kNearestNeighboursBruteForceX(Alloc& a, const Point& p, size_t k, NodeQueue& result, int depth) { double d = Point::distance(p, value_.point()); result.push(this->asNode(), a.convert(this->asNode()), d); if (left_) left(a)->kNearestNeighboursBruteForceX(a, p, k, result, depth + 1); if (right_) right(a)->kNearestNeighboursBruteForceX(a, p, k, result, depth + 1); } template template void SPNode::visit(Alloc& a, Visitor& v, int depth) { v.enter(value_.point(), !left_ && !right_, depth); if (left_) left(a)->visit(a, v, depth + 1); if (right_) right(a)->visit(a, v, depth + 1); v.leave(value_.point(), !left_ && !right_, depth); } template void SPNode::linkNodes(Alloc& a, Node*& prev) { if (prev) { prev->next(a, this->asNode()); } prev = this->asNode(); if (left_) left(a)->linkNodes(a, prev); if (right_) right(a)->linkNodes(a, prev); } template typename SPNode::NodeList SPNode::kNearestNeighboursBruteForce(Alloc& a, const Point& p, size_t k) { NodeQueue queue(k); NodeList result; asNode()->kNearestNeighboursBruteForceX(a, p, k, queue, 0); queue.fill(result); return result; } //---------------------------------------------------------------------------------------------------------------------- template typename SPNode::NodeList SPNode::findInSphere(Alloc& a, const Point& p, double radius) { NodeList result; asNode()->findInSphereX(a, p, radius, result, 0); std::sort(result.begin(), result.end()); return result; } template void SPNode::findInSphereBruteForceX(Alloc& a, const Point& p, double radius, NodeList& result, int depth) { double d = Point::distance(p, value_.point()); if (d <= radius) { result.push_back(NodeInfo(this->asNode(), a.convert(this->asNode()), d)); } if (left_) left(a)->findInSphereBruteForceX(a, p, radius, result, depth + 1); if (right_) right(a)->findInSphereBruteForceX(a, p, radius, result, depth + 1); } template typename SPNode::NodeList SPNode::findInSphereBruteForce(Alloc& a, const Point& p, double radius) { NodeList result; asNode()->findInSphereBruteForceX(a, p, radius, result, 0); std::sort(result.begin(), result.end()); return result; } } // namespace eckit #endif eckit-2.0.7/src/eckit/container/sptree/SPNodeQueue.h0000664000175000017500000000343215161702250022464 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SPNodeQueue_H #define SPNodeQueue_H #include #include #include namespace eckit { template class SPNode; template class SPIterator; template class SPValue; template struct SPNodeInfo; template class SPNodeQueue { public: using Point = typename Traits::Point; using Payload = typename Traits::Payload; using Alloc = typename Traits::Alloc; using ID = typename Alloc::Ptr; using Node = NodeType; using NodeInfo = SPNodeInfo; using NodeList = typename NodeInfo::NodeList; private: size_t k_; std::priority_queue queue_; public: SPNodeQueue(size_t k) : k_(k) {} void push(Node* n, ID id, double d) { NodeInfo info(n, id, d); queue_.push(info); while (queue_.size() > k_) { queue_.pop(); } } double largest() const { return queue_.size() ? queue_.top().distance_ : std::numeric_limits::max(); } void fill(NodeList& v) { v.reserve(k_); while (!queue_.empty()) { v.push_back(queue_.top()); queue_.pop(); } std::sort(v.begin(), v.end()); } bool incomplete() const { return queue_.size() < k_; } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/sptree/SPNode.h0000664000175000017500000000701515161702250021460 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SPNode_H #define SPNode_H #include "eckit/eckit.h" #include "eckit/container/sptree/SPNodeInfo.h" #include "eckit/container/sptree/SPNodeQueue.h" #include "eckit/container/sptree/SPValue.h" namespace eckit { template class SPNode { public: using Point = typename Traits::Point; using Payload = typename Traits::Payload; using Alloc = typename Traits::Alloc; using NodeInfo = SPNodeInfo; using NodeList = typename NodeInfo::NodeList; using Value = SPValue; using NodeQueue = SPNodeQueue; using Node = NodeType; protected: Value value_; using Ptr = typename Alloc::Ptr; Ptr left_; Ptr right_; Ptr next_; // For fast transversal friend struct SPMemory; public: // SPNode(const Value& value); template SPNode(const V& value); ~SPNode() {} NodeInfo nearestNeighbour(Alloc& a, const Point& p); NodeList findInSphere(Alloc& a, const Point& p, double radius); NodeList kNearestNeighbours(Alloc& a, const Point& p, size_t k); const Point& point() const { return value_.point(); } const Payload& payload() const { return value_.payload(); } Value& value() { return value_; } const Value& value() const { return value_; } template static SPNode* build(Alloc& a, const ITER& begin, const ITER& end, int depth = 0); // For testing only NodeInfo nearestNeighbourBruteForce(Alloc& a, const Point& p); NodeList findInSphereBruteForce(Alloc& a, const Point& p, double radius); NodeList kNearestNeighboursBruteForce(Alloc& a, const Point& p, size_t k); // ------- template void visit(Alloc& a, Visitor& v, int depth = 0); // --------- void linkNodes(Alloc& a, Node*& prev = 0); const Node* asNode() const { return static_cast(this); } Node* asNode() { return static_cast(this); } public: // because of a clang bug. Should be protected // void nearestNeighbourX(Alloc& a,const Point& p, Node*& best, double& max, int depth) = 0; void nearestNeighbourBruteForceX(Alloc& a, const Point& p, Node*& best, double& max, int depth); // void findInSphereX(Alloc& a,const Point& p ,double radius, NodeList& result, int depth) = 0; void findInSphereBruteForceX(Alloc& a, const Point& p, double radius, NodeList& result, int depth); // void kNearestNeighboursX(Alloc& a,const Point& p ,size_t k, NodeQueue& result, int depth) = 0; void kNearestNeighboursBruteForceX(Alloc& a, const Point& p, size_t k, NodeQueue& result, int depth); //========================== public: Node* left(Alloc& a) const { return a.convert(left_, (Node*)0); } Node* right(Alloc& a) const { return a.convert(right_, (Node*)0); } Node* next(Alloc& a) const { return a.convert(next_, (Node*)0); } void left(Alloc& a, Node* n) { left_ = a.convert(n); } void right(Alloc& a, Node* n) { right_ = a.convert(n); } void next(Alloc& a, Node* n) { next_ = a.convert(n); } friend class SPIterator; }; } // namespace eckit #include "SPNode.cc" #endif eckit-2.0.7/src/eckit/container/BSPTree.h0000664000175000017500000000347515161702250020300 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef BSPTree_H #define BSPTree_H #include "eckit/container/bsptree/BSPNode.h" #include "eckit/container/sptree/SPTree.h" #include "KDMapped.h" #include "KDMemory.h" namespace eckit { template class BSPTreeX : public SPTree > { public: using Alloc = typename Traits::Alloc; private: Partition partition_; public: BSPTreeX(Alloc& alloc) : SPTree >(alloc) {} /// Container must be a random access /// WARNING: container is changed (sorted) template void build(Container& nodes) { this->root_ = this->alloc_.convert(BSPNode::build(this->alloc_, partition_, nodes, 0.0)); this->alloc_.root(this->root_); } }; template class BSPTreeMemory : public BSPTreeX, Partition> { KDMemory alloc_; public: BSPTreeMemory() : BSPTreeX, Partition>(alloc_) {} }; template class BSPTreeMapped : public BSPTreeX, Partition> { KDMapped alloc_; public: BSPTreeMapped(const eckit::PathName& path, size_t itemCount, size_t metadataSize) : BSPTreeX, Partition>(alloc_), alloc_(path, itemCount, sizeof(BSPNode, Partition>), metadataSize) {} }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/CacheLRU.cc0000664000175000017500000001011015161702250020540 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/container/CacheLRU.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template CacheLRU::CacheLRU(size_t capacity, purge_handler_type purge) : capacity_(capacity), purge_(purge) {} template CacheLRU::~CacheLRU() { clear(); } template bool CacheLRU::insert(const key_type& key, const value_type& value) { bool existed = false; typename map_type::iterator itr = map_.find(key); if (itr != map_.end()) { existed = true; // erase the key from where it is // we'll reinsert it again so it comes on top erase(itr); } storage_.push_front(Entry(key, value)); map_[key] = storage_.begin(); trim(); return existed; } template V CacheLRU::access(const key_type& key) { // check first the front() since it is the most popular/recent entry if (size() && storage_.front().key_ == key) return storage_.front().value_; typename map_type::iterator itr = map_.find(key); if (itr != map_.end()) { // move entry of list to front // this keeps the most popular in front moveToFront(itr); return valueFrom(itr); } else { throw eckit::OutOfRange("key not in CacheLRU", Here()); } } template V CacheLRU::extract(const key_type& key) { typename map_type::iterator itr = map_.find(key); if (itr == map_.end()) { throw OutOfRange("key not in CacheLRU", Here()); } value_type result = valueFrom(itr); erase(itr); return result; } template bool CacheLRU::remove(const key_type& key) { bool existed = false; typename map_type::iterator itr = map_.find(key); if (itr != map_.end()) { existed = true; purge(itr->second->key_, valueFrom(itr)); erase(itr); } return existed; } template bool CacheLRU::exists(const key_type& key) const { return (map_.find(key) != map_.end()); } template void CacheLRU::clear() { for (storage_iterator itr = storage_.begin(); itr != storage_.end(); ++itr) { purge(itr->key_, itr->value_); } storage_.clear(); map_.clear(); } template void CacheLRU::erase(typename map_type::iterator itr) { storage_.erase(itr->second); map_.erase(itr); } template void CacheLRU::trim() { while (map_.size() > capacity_) { entry_type entry = storage_.back(); purge(entry.key_, entry.value_); map_.erase(entry.key_); storage_.pop_back(); } } template void CacheLRU::moveToFront(typename map_type::iterator itr) { storage_.splice(storage_.begin(), storage_, itr->second); } template void CacheLRU::purge(key_type& key, value_type& value) const { if (purge_) purge_(key, value); } template void CacheLRU::capacity(size_t size) { capacity_ = size; trim(); } template void CacheLRU::print(std::ostream& os) const { os << "CacheLRU(capacity=" << capacity_ << ",size=" << storage_.size() << ",storage={"; for (typename storage_type::const_iterator itr = storage_.begin(); itr != storage_.end(); ++itr) { os << *itr << ","; } os << "})"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/container/DenseSet.h0000664000175000017500000001165415161702250020544 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_DenseSet_h #define eckit_DenseSet_h /// @author Tiago Quintino /// @author Olivier Iffrig #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/JSON.h" namespace eckit { template class DenseSet { public: // types using value_type = V; ///< value type private: // types using store_t = std::vector; public: // methods using const_reference = typename store_t::const_reference; using iterator = typename store_t::const_iterator; using const_iterator = typename store_t::const_iterator; DenseSet(size_t s = 0) : sorted_(true) { if (s > 0) { reserve(s); } } ~DenseSet() {} void reserve(size_t s) { values_.reserve(s); } void insert(const V& v) { if (!values_.empty() && v <= values_.back()) { sorted_ = false; } values_.push_back(v); } /// @TODO shoudl we implement this? // size_t erase( const value_type& v ); void clear() { values_.clear(); sorted_ = true; } bool sorted() const { return sorted_; } size_t size() const { return values_.size(); } bool empty() const { return values_.empty(); } void sort() { if (!sorted_) { std::sort(values_.begin(), values_.end()); auto last = std::unique(values_.begin(), values_.end()); values_.erase(last, values_.end()); sorted_ = true; } } iterator begin() { return values_.begin(); } const_iterator begin() const { return values_.begin(); } const_iterator cbegin() const { return values_.cbegin(); } iterator end() { return values_.end(); } const_iterator end() const { return values_.end(); } const_iterator cend() const { return values_.cend(); } bool contains(const V& v) const { return find(v) != cend(); } const_reference at(const size_t i) const { ASSERT(i < values_.size()); return values_[i]; } const_reference operator[](const size_t& i) const { return values_[i]; } iterator find(const V& v) { if (empty()) { return end(); } ASSERT(sorted()); iterator it = std::lower_bound(begin(), end(), v); if (it != end() && *it == v) { return it; } return end(); } const_iterator find(const V& v) const { if (empty()) { return cend(); } ASSERT(sorted()); const_iterator it = std::lower_bound(cbegin(), cend(), v); if (it != cend() && *it == v) { return it; } return cend(); } void print(std::ostream& s) const { const_iterator it = cbegin(); for (; it != cend(); ++it) { s << *it << std::endl; } } void json(JSON& s) const { s.startList(); const_iterator it = cbegin(); for (; it != cend(); ++it) { s << *it; } s.endList(); } bool operator==(const DenseSet& rhs) const { return !operator!=(rhs); } bool operator!=(const DenseSet& rhs) const { return values_ != rhs.values_; } friend std::ostream& operator<<(std::ostream& s, const DenseSet& m) { m.print(s); return s; } friend JSON& operator<<(JSON& s, const DenseSet& m) { m.json(s); return s; } void merge(const DenseSet& other) { if (other.empty()) return; if (empty()) { values_ = other.values_; sorted_ = other.sorted_; return; } ASSERT(sorted_); ASSERT(other.sorted_); int i = 0; int j = 0; // n.b. resizes would invalidate iterators --> use indices instead while (j < other.values_.size()) { if (i >= values_.size() || values_[i] > other.values_[j]) { if (values_.capacity() == values_.size()) { values_.reserve(std::max(values_.size(), other.values_.size()) * 2); } values_.insert(values_.begin() + i, other.values_[j]); i++; j++; } else if (values_[i] == other.values_[j]) { j++; } else { // if (*it1 < *it2) i++; } } } private: // members store_t values_; ///< storage of the values bool sorted_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/container/BTree.h0000664000175000017500000001735315161702250020035 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File BTree.h // Baudouin Raoult - (c) ECMWF Feb 12 #ifndef eckit_BTree_h #define eckit_BTree_h #include #include #include #include "eckit/container/BTree.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/PooledFileDescriptor.h" #include "eckit/memory/Padded.h" #include "eckit/os/Stat.h" #include "eckit/thread/AutoLock.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class BTreeLock { public: static void lockRange(int fd, off_t start, off_t len, int cmd, int type) { struct flock lock; lock.l_type = type; lock.l_whence = SEEK_SET; lock.l_start = start; lock.l_len = len; SYSCALL(::fcntl(fd, cmd, &lock)); } }; class BTreeNoLock { public: static void lockRange(int fd, off_t start, off_t len, int cmd, int type) {} }; //---------------------------------------------------------------------------------------------------------------------- /// B+Tree index /// /// @todo Deletion /// @todo Cache pages /// @invariant K and V needs to be PODs /// @invariant S is the page size padding /// @invariant L implements locking policy /// template class BTree { public: using key_type = K; using value_type = V; using result_type = std::pair; // -- Contructors BTree(const PathName&, bool readOnly = false, off_t offset = 0); BTree(const BTree&) = delete; BTree& operator=(const BTree&) = delete; BTree(BTree&&) = delete; BTree& operator=(BTree&&) = delete; // -- Destructor ~BTree(); // -- Methods bool get(const K&, V&); bool set(const K&, const V&); void preload(); template void range(const K& key1, const K& key2, T& result); bool remove(const K&); void dump(std::ostream& s = std::cout) const; /// Counts the entries in the whole tree /// This is not an efficient call since it visits the whole tree. Use with care. size_t count() const; /// Counts the entries in a page of the tree size_t count(unsigned long page) const; void lock(); void lockShared(); void unlock(); void flock(); void funlock(); void flush(); void sync(); const PathName& path() const { return path_; } private: // methods void dump(std::ostream&, unsigned long page, int depth) const; void print(std::ostream& o) const { dump(o); } private: struct _Header {}; struct NodeEntry { K key_; unsigned long page_; bool operator<(const K& key) const { return key_ < key; } bool operator>(const K& key) const { return key_ > key; } bool operator==(const K& key) const { return key_ == key; } bool operator!=(const K& key) const { return key_ != key; } bool operator>=(const K& key) const { return key_ >= key; } bool operator<=(const K& key) const { return key_ <= key; } bool operator<(const NodeEntry& e) const { return key_ < e.key_; } bool operator>(const NodeEntry& e) const { return key_ > e.key_; } bool operator==(const NodeEntry& e) const { return key_ == e.key_; } bool operator!=(const NodeEntry& e) const { return key_ != e.key_; } bool operator>=(const NodeEntry& e) const { return key_ >= e.key_; } bool operator<=(const NodeEntry& e) const { return key_ <= e.key_; } }; struct LeafEntry { K key_; V value_; bool operator<(const K& key) const { return key_ < key; } bool operator>(const K& key) const { return key_ > key; } bool operator==(const K& key) const { return key_ == key; } bool operator!=(const K& key) const { return key_ != key; } bool operator>=(const K& key) const { return key_ >= key; } bool operator<=(const K& key) const { return key_ <= key; } bool operator<(const NodeEntry& e) const { return key_ < e.key_; } bool operator>(const NodeEntry& e) const { return key_ > e.key_; } bool operator==(const NodeEntry& e) const { return key_ == e.key_; } bool operator!=(const NodeEntry& e) const { return key_ != e.key_; } bool operator>=(const NodeEntry& e) const { return key_ >= e.key_; } bool operator<=(const NodeEntry& e) const { return key_ <= e.key_; } }; struct _Page { unsigned long id_; unsigned long count_; unsigned long node_; unsigned long left_; unsigned long right_; }; struct _LeafPage : public _Page { static const size_t SIZE = (S - sizeof(_Page)) / sizeof(LeafEntry); LeafEntry lentries_[SIZE]; void print(std::ostream& s) const; }; struct _NodePage : public _Page { static const size_t SIZE = (S - sizeof(_Page)) / sizeof(NodeEntry); NodeEntry nentries_[SIZE]; void print(std::ostream& s) const; }; struct NodePage : Padded<_NodePage, S> {}; struct LeafPage : Padded<_LeafPage, S> {}; struct Page : Padded<_Page, S> { NodePage& nodePage() { return *(reinterpret_cast(this)); } LeafPage& leafPage() { return *(reinterpret_cast(this)); } const NodePage& nodePage() const { return *(reinterpret_cast(this)); } const LeafPage& leafPage() const { return *(reinterpret_cast(this)); } void print(std::ostream& s) const; friend std::ostream& operator<<(std::ostream& s, const Page& p) { p.print(s); return s; } }; static const size_t maxNodeEntries_ = _NodePage::SIZE; // split at full page -- could be a percentage static const size_t maxLeafEntries_ = _LeafPage::SIZE; // split at full page -- could be a percentage PathName path_; mutable PooledFileDescriptor file_; bool cacheReads_; bool cacheWrites_; bool readOnly_; off_t offset_; struct _PageInfo { Page* page_; unsigned long long count_; time_t last_; bool dirty_; _PageInfo(Page* page = 0) : page_(page), count_(0), last_(time(nullptr)), dirty_(false) {} }; using Cache = std::map; Cache cache_; void lockRange(off_t start, off_t len, int cmd, int type); bool search(unsigned long page, const K&, V&) const; template void search(unsigned long page, const K& key1, const K& key2, T& result); void splitRoot(); off_t pageOffset(unsigned long) const; void savePage(const Page&); void loadPage(unsigned long, Page&) const; void newPage(Page&); void _savePage(const Page&); void _loadPage(unsigned long, Page&) const; void _newPage(Page&); bool insert(unsigned long page, const K& key, const V& value, std::vector& path); bool store(unsigned long page, const K& key, const V& value, std::vector& path); unsigned long next(const K&, const Page&) const; // -- Friends friend std::ostream& operator<<(std::ostream& s, const BTree& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #include "BTree.cc" #endif eckit-2.0.7/src/eckit/container/Recycler.h0000664000175000017500000001044415161702250020576 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Recycler.h // Baudouin Raoult - ECMWF Apr 97 #ifndef eckit_Recycler_h #define eckit_Recycler_h #include #include #include #include "eckit/eckit.h" #include "eckit/container/Recycler.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/thread/AutoLock.h" #include "eckit/types/Types.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- template class Recycler { public: // -- Exceptions // None // -- Contructors Recycler(const PathName&); // -- Destructor ~Recycler(); // -- Convertors // None // -- Operators // None // -- Methods void lock(); void unlock(); void push(const T&); template void push(Iter begin, Iter end); bool pop(T& value); template Ordinal pop(Iter begin, Ordinal count); // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed Recycler(const Recycler&); Recycler& operator=(const Recycler&); // -- Members // None PathName path_; int fd_; // -- Methods // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends friend std::ostream& operator<<(std::ostream& s, const Recycler& p) { p.print(s); return s; } }; //----------------------------------------------------------------------------- template Recycler::Recycler(const PathName& path) : path_(path), fd_(-1) { path_.dirName().mkdir(); fd_ = ::open(path_.localPath(), O_RDWR | O_CREAT, 0777); if (fd_ < 0) { throw CantOpenFile(path_); } } template Recycler::~Recycler() { if (fd_ >= 0) { SYSCALL(::close(fd_)); } } template void Recycler::lock() { struct flock lock; lock.l_type = F_WRLCK; lock.l_whence = SEEK_SET; lock.l_start = 0; lock.l_len = 0; SYSCALL(::fcntl(fd_, F_SETLK, &lock)); } template void Recycler::unlock() { struct flock lock; lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = 0; lock.l_len = 0; SYSCALL(::fcntl(fd_, F_SETLK, &lock)); } template template void Recycler::push(Iter begin, Iter end) { AutoLock > lock(this); off_t here; SYSCALL(here = ::lseek(fd_, 0, SEEK_END)); ASSERT((here % sizeof(T)) == 0); for (Iter j = begin; j != end; ++j) { ASSERT(::write(fd_, &(*j), sizeof(T)) == sizeof(T)); } } template template Ordinal Recycler::pop(Iter begin, Ordinal count) { AutoLock > lock(this); off_t here, there; SYSCALL(here = ::lseek(fd_, 0, SEEK_END)); ASSERT((here % sizeof(T)) == 0); Ordinal cnt = std::min(Ordinal(here / sizeof(T)), count); here -= cnt * sizeof(T); SYSCALL(there = ::lseek(fd_, here, SEEK_SET)); ASSERT(there == here); for (Ordinal i = 0; i < cnt; i++) { T value; ASSERT(::read(fd_, &value, sizeof(T)) == sizeof(T)); *(begin++) = value; } SYSCALL(::ftruncate(fd_, here)); return cnt; } template bool Recycler::pop(T& value) { return pop(&value, 1) == 1; } template void Recycler::push(const T& value) { const T* p = &value; push(p, p + 1); } //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/container/kdtree/0000775000175000017500000000000015161702250020130 5ustar alastairalastaireckit-2.0.7/src/eckit/container/kdtree/KDNode.h0000664000175000017500000000350415161702250021407 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef KDNode_H #define KDNode_H #include "eckit/container/sptree/SPNode.h" namespace eckit { template class KDNode : public SPNode> { public: using SPNodeType = SPNode>; // cannot redefine as SPNode since some compilers in-class redefinitions using Value = typename SPNodeType::Value; using Alloc = typename SPNodeType::Alloc; using Point = typename SPNodeType::Point; using NodeList = typename SPNodeType::NodeList; using NodeQueue = typename SPNodeType::NodeQueue; using NodeInfo = typename SPNodeType::NodeInfo; using Node = KDNode; private: size_t axis_; public: KDNode(const Value& value, size_t axis); ~KDNode() {} template static KDNode* build(Alloc& a, const ITER& begin, const ITER& end, int depth = 0); static KDNode* insert(Alloc& a, const Value& value, KDNode* node, int depth = 0); /// Return the axis along which this node is split. size_t axis() const { return axis_; } public: void nearestNeighbourX(Alloc& a, const Point& p, Node*& best, double& max, int depth); void findInSphereX(Alloc& a, const Point& p, double radius, NodeList& result, int depth); void kNearestNeighboursX(Alloc& a, const Point& p, size_t k, NodeQueue& result, int depth); //========================== }; } // namespace eckit #include "KDNode.cc" #endif eckit-2.0.7/src/eckit/container/kdtree/KDNode.cc0000664000175000017500000001306215161702250021545 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef KDNode_CC #define KDNode_CC #include #include #include #include #include #include #include #include "KDNode.h" namespace eckit { template KDNode::KDNode(const Value& value, size_t axis) : SPNodeType(value), axis_(axis) {} template void KDNode::nearestNeighbourX(Alloc& a, const Point& p, Node*& best, double& max, int depth) { a.statsVisitNode(); bool left_visited = false; bool right_visited = false; if (p.x(axis_) < this->value_.point().x(axis_)) { if (this->left_) { this->left(a)->nearestNeighbourX(a, p, best, max, depth + 1); left_visited = true; } } else { if (this->right_) { this->right(a)->nearestNeighbourX(a, p, best, max, depth + 1); right_visited = true; } } double d = Point::distance(p, this->value_.point()); if (d < max) { max = d; best = this; a.statsNewCandidateOK(); } else { a.statsNewCandidateMiss(); } d = Point::distance(p, this->value_.point(), axis_); if (d < max) { // Visit other subtree... a.statsCrossOver(); if (p.x(axis_) < this->value_.point().x(axis_)) { if (this->right_ && !right_visited) { this->right(a)->nearestNeighbourX(a, p, best, max, depth + 1); } } else { if (this->left_ && !left_visited) { this->left(a)->nearestNeighbourX(a, p, best, max, depth + 1); } } } } //---------------------------------------------------------------------------------------------------------------------- template void KDNode::kNearestNeighboursX(Alloc& a, const Point& p, size_t k, NodeQueue& result, int depth) { if (p.x(axis_) < this->value_.point().x(axis_)) { if (this->left_) this->left(a)->kNearestNeighboursX(a, p, k, result, depth + 1); } else { if (this->right_) this->right(a)->kNearestNeighboursX(a, p, k, result, depth + 1); } double d = Point::distance(p, this->value_.point()); Node* self = this; result.push(self, a.convert(self), d); if (Point::distance(p, this->value_.point(), axis_) <= result.largest()) { // Visit other subtree... a.statsCrossOver(); if (p.x(axis_) < this->value_.point().x(axis_)) { if (this->right_) this->right(a)->kNearestNeighboursX(a, p, k, result, depth + 1); } else { if (this->left_) this->left(a)->kNearestNeighboursX(a, p, k, result, depth + 1); } } } template struct sorter { int axis_; bool operator()(const Value& a, const Value& b) { return (a.point().x(axis_) < b.point().x(axis_)); } sorter(size_t axis) : axis_(axis) {} }; template template KDNode* KDNode::build(Alloc& a, const ITER& begin, const ITER& end, int depth) { if (end == begin) return 0; a.statsDepth(depth); // size_t k = Point::size(*begin); size_t k = Point::DIMS; size_t axis = depth % k; // std::sort(begin, end, sorter(axis)); size_t median = (end - begin) / 2; std::nth_element(begin, begin + median, end, sorter(axis)); ITER e2 = begin + median; ITER b2 = begin + median + 1; KDNode* n = a.newNode2(*e2, axis, (KDNode*)0); n->left(a, build(a, begin, e2, depth + 1)); n->right(a, build(a, b2, end, depth + 1)); return n; } template KDNode* KDNode::insert(Alloc& a, const Value& value, KDNode* node, int depth) { // size_t k = Point::size(*begin); size_t k = Point::DIMS; size_t axis = depth % k; if (node == 0) { return a.newNode2(value, axis, (KDNode*)0); } if (value.point().x(axis) <= node->value_.point().x(axis)) { node->left(a, insert(a, value, node->left(a), depth + 1)); } else { node->right(a, insert(a, value, node->right(a), depth + 1)); } return node; } template void KDNode::findInSphereX(Alloc& a, const Point& p, double radius, NodeList& result, int depth) { if (p.x(axis_) < this->value_.point().x(axis_)) { if (this->left_) this->left(a)->findInSphereX(a, p, radius, result, depth + 1); } else { if (this->right_) this->right(a)->findInSphereX(a, p, radius, result, depth + 1); } double d = Point::distance(p, this->value_.point()); if (d <= radius) { result.push_back(NodeInfo(this, a.convert(this), d)); } if (Point::distance(p, this->value_.point(), axis_) <= radius) { // Visit other subtree... if (p.x(axis_) < this->value_.point().x(axis_)) { if (this->right_) this->right(a)->findInSphereX(a, p, radius, result, depth + 1); } else { if (this->left_) this->left(a)->findInSphereX(a, p, radius, result, depth + 1); } } } } // namespace eckit #endif eckit-2.0.7/src/eckit/container/DenseMap.h0000664000175000017500000001176515161702250020531 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_DenseMap_h #define eckit_DenseMap_h /// @author Tiago Quintino #include #include #include #include "eckit/exception/Exceptions.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- template class DenseMap { public: // types using key_type = K; ///< key type using value_type = V; ///< value type using item_type = std::pair; ///< (key, value) item type private: // types using store_t = std::deque; public: // methods using iterator = typename store_t::iterator; using const_iterator = typename store_t::const_iterator; DenseMap() : sorted_(true) {} ~DenseMap() {} void insert(const K& k, const V& v) { items_.push_back(std::make_pair(k, v)); sorted_ = false; } /// @TODO shoudl we implement this? // size_t erase( const key_type& k ); void replace(const K& k, const V& v) { iterator it = find(k); if (it != end()) { it->second = v; } else { insert(k, v); } } void clear() { items_.clear(); sorted_ = true; } bool sorted() const { return sorted_; } size_t size() const { ASSERT(sorted_); return items_.size(); } bool empty() const { return items_.empty(); } void sort() { if (sorted_) { return; } std::stable_sort(items_.begin(), items_.end(), LessThan()); auto last = std::unique(items_.rbegin(), items_.rend(), Equals()); auto position = std::distance(last, items_.rend()); items_.erase(items_.begin(), items_.begin() + position); sorted_ = true; } iterator begin() { return items_.begin(); } const_iterator begin() const { return items_.begin(); } const_iterator cbegin() const { return items_.cbegin(); } iterator end() { return items_.end(); } const_iterator end() const { return items_.end(); } const_iterator cend() const { return items_.cend(); } bool contains(const K& k) const { return find(k) != cend(); } const value_type& get(const K& k) const { return find(k)->second; } value_type& get(const K& k) { return find(k)->second; } const value_type& at(const size_t i) const { ASSERT(i < items_.size()); return items_[i].second; } value_type& at(const size_t i) { ASSERT(i < items_.size()); return items_[i].second; } const value_type& operator[](const K& k) const { return find(k)->second; } value_type& operator[](const K& k) { return find(k)->second; } const value_type& operator[](const size_t& i) const { return items_[i].second; } value_type& operator[](const size_t& i) { return items_[i].second; } iterator find(const K& k) { if (empty()) { return end(); } ASSERT(sorted()); iterator it = std::lower_bound(begin(), end(), k, Compare()); if (it != end() && it->first == k) { return it; } return end(); } const_iterator find(const K& k) const { if (empty()) { return cend(); } ASSERT(sorted()); const_iterator it = std::lower_bound(cbegin(), cend(), k, Compare()); if (it != cend() && it->first == k) { return it; } return cend(); } void print(std::ostream& s) const { const_iterator it = cbegin(); for (; it != cend(); ++it) { s << it->first << " " << it->second << std::endl; } } friend std::ostream& operator<<(std::ostream& s, const DenseMap& m) { m.print(s); return s; } private: // types class LessThan { public: bool operator()(const item_type& e1, const item_type& e2) const { return (e1.first < e2.first) ? true : false; } }; class Equals { public: bool operator()(const item_type& e1, const item_type& e2) const { return (e1.first == e2.first) ? true : false; } }; class Compare { public: bool operator()(const item_type& e, const K& k) const { return (e.first < k) ? true : false; } bool operator()(const K& k, const item_type& e) const { return (e.first > k) ? true : false; } }; private: // members store_t items_; ///< storage of the items bool sorted_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/container/CacheLRU.h0000664000175000017500000000713515161702250020417 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @date Dec 2015 #ifndef eckit_container_CacheLRU_h #define eckit_container_CacheLRU_h #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/CodeLocation.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template class CacheLRU { public: // types using key_type = K; using value_type = V; struct Entry { key_type key_; value_type value_; Entry(const key_type& k, const value_type& v) : key_(k), value_(v) {} friend std::ostream& operator<<(std::ostream& s, const Entry& e) { s << "key=" << e.key_; return s; } }; using entry_type = Entry; using storage_type = std::list; using storage_iterator = typename storage_type::iterator; using map_type = std::map; using purge_handler_type = void (*)(key_type&, value_type&); public: // methods CacheLRU(size_t capacity, purge_handler_type purge = 0); CacheLRU(const CacheLRU&) = delete; CacheLRU& operator=(const CacheLRU&) = delete; CacheLRU(CacheLRU&&) = delete; CacheLRU& operator=(CacheLRU&&) = delete; ~CacheLRU(); /// Inserts an entry into the cache, overwrites if already exists /// @returns true if a key already existed bool insert(const key_type& key, const value_type& value); /// Accesses a key that must already exist /// @throws OutOfRange exception is key not in cache value_type access(const key_type& key); /// Extracts the key from the cache without purging /// @pre Key must exist in cache /// @throws OutOfRange exception if key not in cache value_type extract(const key_type& key); /// Remove a key-value pair from the cache /// No effect if key is not present /// /// @return true if removed bool remove(const key_type& key); /// @returns true if the key exists in the cache bool exists(const key_type& key) const; /// Clears all entries in the cache void clear(); /// @returns the maximum size of the cache size_t capacity() const { return capacity_; } /// @returns the current (used) size of the cache size_t size() const { return storage_.size(); } /// resizes the cache capacity void capacity(size_t size); void print(std::ostream& os) const; friend std::ostream& operator<<(std::ostream& s, const CacheLRU& p) { p.print(s); return s; } private: // methods void erase(typename map_type::iterator itr); void trim(); void moveToFront(typename map_type::iterator itr); value_type& valueFrom(typename map_type::iterator itr) const { return itr->second->value_; } void purge(key_type& key, value_type& value) const; private: // members storage_type storage_; map_type map_; size_t capacity_; purge_handler_type purge_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #include "CacheLRU.cc" #endif eckit-2.0.7/src/eckit/container/Queue.h0000664000175000017500000001075115161702250020113 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @author Simon Smart /// @date June 2018 #ifndef eckit_container_Queue_h #define eckit_container_Queue_h #include #include #include #include #include #include "eckit/exception/Exceptions.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class QueueInterruptedError : public Exception { public: QueueInterruptedError(const std::string& msg, const CodeLocation& here) : Exception(std::string("Threaded queue interrupted") + (msg.size() ? (std::string(": ") + msg) : std::string()), here) {} }; //---------------------------------------------------------------------------------------------------------------------- template class Queue { public: // public Queue(size_t max) : max_(max), interrupt_{nullptr}, closed_(false) { ASSERT(max > 0); } Queue(const Queue&) = delete; Queue& operator=(const Queue&) = delete; // n.b. cannot move object with std::condition_variable Queue(Queue&& rhs) = delete; Queue& operator=(Queue&& rhs) = delete; size_t maxSize() const { return max_; } void resize(size_t size) { ASSERT(size > 0); std::unique_lock locker(mutex_); while (checkInterrupt() && queue_.size() > size) { cv_.wait(locker); } ASSERT(!closed_); max_ = size; cv_.notify_all(); } void close() { std::unique_lock locker(mutex_); closed_ = true; cv_.notify_all(); } bool closed() { std::unique_lock locker(mutex_); return closed_ || interrupt_; } bool empty() { std::unique_lock locker(mutex_); return queue_.empty(); } bool checkInterrupt() { if (interrupt_) { std::rethrow_exception(interrupt_); } return true; } void interrupt(std::exception_ptr expn) { std::unique_lock locker(mutex_); interrupt_ = expn; cv_.notify_all(); } long pop(ELEM& e) { std::unique_lock locker(mutex_); while (checkInterrupt() && queue_.empty()) { if (closed_) { return -1; } cv_.wait(locker); } std::swap(e, queue_.front()); queue_.pop(); size_t size = queue_.size(); cv_.notify_all(); return long(size); } long pop(std::vector& elems) { std::unique_lock locker(mutex_); while (checkInterrupt() && queue_.empty()) { if (closed_) { return -1; } cv_.wait(locker); } long count = std::min(elems.size(), queue_.size()); for (long i = 0; i < count; i++) { std::swap(elems[i], queue_.front()); queue_.pop(); } cv_.notify_all(); return count; } size_t push(const ELEM& e) { std::unique_lock locker(mutex_); while (checkInterrupt() && queue_.size() >= max_) { cv_.wait(locker); } ASSERT(!closed_); queue_.push(e); size_t size = queue_.size(); cv_.notify_all(); return size; } template size_t emplace(Args&&... args) { std::unique_lock locker(mutex_); while (checkInterrupt() && queue_.size() >= max_) { cv_.wait(locker); } ASSERT(!closed_); queue_.emplace(std::forward(args)...); size_t size = queue_.size(); cv_.notify_all(); return size; } private: // members std::queue queue_; std::mutex mutex_; std::condition_variable cv_; size_t max_; std::exception_ptr interrupt_; bool closed_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif // eckit_container_Queue_h eckit-2.0.7/src/eckit/container/Cache.h0000664000175000017500000001673415161702250020041 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @date Oct 2012 #ifndef eckit_container_Cache_h #define eckit_container_Cache_h #include #include #include #include #include #include "eckit/eckit.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- /// @todo make an apply() method /// @todo implement the expire() and the different policies template class Cache { public: // types struct Entry { Entry(const V& v) : v_(v), expired_(false), hits_(0) { gettimeofday(&age_, nullptr); last_ = age_; } void reset(const V& v) { v_ = v; expired_ = false; hits_ = 0; gettimeofday(&age_, nullptr); last_ = age_; } V& access() { gettimeofday(&last_, nullptr); ++hits_; return v_; } V v_; bool expired_; uint64_t hits_; struct ::timeval age_; struct ::timeval last_; }; using key_type = K; using value_type = V; using entry_type = Entry; using store_type = std::map; class Policy { /// Expires the Least Recently Used (LRU) entries /// @returns true if any entries were expired static bool expireLRU(store_type& c, const size_t& maxSize); /// Expires the Least Frequently Used (LFU) entries /// @returns true if any entries were expired static bool expireLFU(store_type& c, const size_t& maxSize); /// Expires the entries older than a certain age /// @returns true if any entries were expired static bool expireAge(store_type& c, struct ::timeval& maxAge); }; public: // methods Cache(); Cache(const Cache&) = delete; Cache& operator=(const Cache&) = delete; Cache(Cache&&) = delete; Cache& operator=(Cache&&) = delete; ~Cache(); /// inserts an object in the cache /// @returns true if object was correctly inserted, /// or false if key already existed and cannot be inserted bool insert(const K&, const V&); /// updates an object in the cache (or inserts if does not exist) /// @returns true if object existed and was updated, /// false if the object did not exist and was inserted bool update(const K&, const V&); /// accesses an object in the cache /// @param v returns the object bool fetch(const K&, V&); /// marks an object as expired /// @returns true if object was present and is marked as expired bool expire(const K&); /// evicts entries that are considered expired void purge(); /// evicts all entries void clear(); /// @returns true if entry exists and is not expired in cache bool valid(const K&) const; /// @returns the number of entries in the cache, expired or not size_t size() const; void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const Cache& p) { p.print(s); return s; } private: // methods /// marks an object as expired /// @returns true if object was present and is marked as expired void expire(typename store_type::iterator i); private: // members store_type storage_; }; //----------------------------------------------------------------------------- template Cache::Cache() : storage_() {} template Cache::~Cache() { clear(); } template bool Cache::insert(const K& k, const V& v) { typename store_type::iterator i = storage_.find(k); if (i != storage_.end()) { Entry& e = i->second; if (!e.expired_) { return false; } e.reset(v); } else { storage_.insert(std::make_pair(k, Entry(v))); } return true; } template bool Cache::update(const K& k, const V& v) { typename store_type::iterator i = storage_.find(k); if (i != storage_.end()) { Entry& e = i->second; e.reset(v); return true; } storage_.insert(std::make_pair(k, Entry(v))); return false; } template bool Cache::fetch(const K& k, V& v) { typename store_type::iterator i = storage_.find(k); if (i != storage_.end()) { Entry& e = i->second; if (!e.expired_) { v = e.access(); return true; } } return false; } template bool Cache::expire(const K& k) { typename store_type::iterator i = storage_.find(k); if (i != storage_.end()) { this->expire(i); return true; } return false; } template void Cache::expire(typename store_type::iterator i) { Entry& e = i->second; e.expired_ = true; } template void Cache::purge() { // collect all expired using siterator = typename store_type::iterator; std::vector expired; for (siterator i = storage_.begin(); i != storage_.end(); ++i) { if (i->second.expired_) { expired.push_back(i); } } // remove them for (typename std::vector::iterator e = expired.begin(); e != expired.end(); ++e) { storage_.erase(*e); } } template void Cache::clear() { storage_.clear(); } template bool Cache::valid(const K& k) const { typename store_type::const_iterator i = storage_.find(k); if (i != storage_.end() && !i->second.expired_) { return true; } return false; } template size_t Cache::size() const { return storage_.size(); } template void Cache::print(std::ostream& out) const { using siterator = typename store_type::const_iterator; for (siterator i = storage_.begin(); i != storage_.end(); ++i) { out << i->second.v_ << std::endl; } } //----------------------------------------------------------------------------- template bool Cache::Policy::expireLRU(typename Cache::store_type& c, const size_t& maxSize) { // typedef std::vector< std::pair< // typedef typename store_type::iterator siterator; // for( siterator i = storage_.begin(); i != storage_.end(); ++i ) // bool found = false; // typedef typename store_type::iterator siterator; // for( siterator i = storage_.begin(); i != storage_.end(); ++i ) // { // if( !i->second.expired_ && policy_->toExpire( i->second ) ) // { // Entry& e = i->second; // e.expired_ = true; // found = true; // } // } // return found; return false; } //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/container/BTree.cc0000664000175000017500000004646615161702250020202 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #ifdef __linux__ #include #ifndef ENOTSUPP #define ENOTSUPP 524 #endif #endif #include "eckit/exception/Exceptions.h" #include "eckit/io/FDataSync.h" #include "eckit/memory/Zero.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template void BTree::Page::print(std::ostream& s) const { s << ((this->node_) ? "NODE" : "LEAF") << "_PAGE[id=" << this->id_ << ",count=" << this->count_ << "]"; // For some strange reason "this" is required... if (this->node_) { this->nodePage().print(s); } else { this->leafPage().print(s); } } template void BTree::_LeafPage::print(std::ostream& s) const { // For some strange reason "this" is required... s << "("; for (unsigned long i = 0; i < this->count_; i++) { s << lentries_[i].key_ << ":" << lentries_[i].value_ << ","; } s << ")"; s << "[" << this->left_ << "<=>" << this->right_ << "]"; } template void BTree::_NodePage::print(std::ostream& s) const { // For some strange reason "this" is required... s << "{" << this->left_ << "!"; for (unsigned long i = 0; i < this->count_; i++) { s << nentries_[i].key_ << "@" << nentries_[i].page_ << ","; } s << "}"; } template BTree::BTree(const PathName& path, bool readOnly, off_t offset) : path_(path), file_(path, readOnly), cacheReads_(true), cacheWrites_(true), readOnly_(readOnly), offset_(offset) { file_.open(); AutoLock > lock(this); off_t here = file_.seekEnd(); if (here <= offset_) { here = file_.seek(offset_); // Add root page Page root; newPage(root); ASSERT(root.id_ == 1); } else { // TODO: Check header } static_assert(maxLeafEntries_ > 3, "maxLeafEntries_ > 3"); static_assert(maxNodeEntries_ > 3, "maxNodeEntries_ > 3"); static_assert(sizeof(Page) == sizeof(LeafPage), "sizeof Page must be equal to sizeof LeafPage"); static_assert(sizeof(Page) == sizeof(NodePage), "sizeof Page must be equal to sizeof NodePage"); // std::cout << "::BTree : maxLeafEntries_=" << maxLeafEntries_ << ", maxNodeEntries_=" << // maxNodeEntries_ << std::endl; } template void BTree::preload() { Page p; loadPage(1, p); while (p.right_) { loadPage(p.right_, p); } } template BTree::~BTree() { if (file_.fileno() >= 0) { flush(); file_.close(); } for (typename Cache::iterator j = cache_.begin(); j != cache_.end(); ++j) delete (*j).second.page_; } template void BTree::flush() { for (typename Cache::iterator j = cache_.begin(); j != cache_.end(); ++j) { // Log::info() << "BTree::flush() " << path_ << " " << (*j).first << ", " << // (*j).second.dirty_ << std::endl; if ((*j).second.dirty_) { _savePage(*(*j).second.page_); (*j).second.dirty_ = false; } } } template void BTree::sync() { file_.sync(); } template void BTree::dump(std::ostream& s, unsigned long page, int depth) const { Page p; loadPage(page, p); for (int i = 0; i < depth; i++) s << " "; s << p << std::endl; if (p.node_) { dump(s, p.left_, depth + 1); for (unsigned long i = 0; i < p.count_; i++) dump(s, p.nodePage().nentries_[i].page_, depth + 1); } } template void BTree::dump(std::ostream& s) const { AutoSharedLock > lock(const_cast(this)); s << "::BTree : maxLeafEntries_=" << maxLeafEntries_ << ", maxNodeEntries_=" << maxNodeEntries_ << std::endl; dump(s, 1, 0); } template bool BTree::set(const K& key, const V& value) { AutoLock > lock(this); // std::cout << "Set " << key << " -> " << value << std::endl; std::vector path; return insert(1, key, value, path); } template unsigned long BTree::next(const K& key, const Page& p) const { ASSERT(p.node_); const NodeEntry* begin = p.nodePage().nentries_; const NodeEntry* end = begin + p.count_; ASSERT(begin != end); if (key < (*begin).key_) { // std::cout << "next " << key << " in " << p << " " << p.left_ << " (FIRST)" << std::endl; return p.left_; } const NodeEntry* e = std::lower_bound(begin, end, key); if (e == end || key < (*e).key_) { e--; } // std::cout << "next " << key << " -> " << (*e).key_ << ":" << (*e).page_ << std::endl; return (*e).page_; } template bool BTree::insert(unsigned long page, const K& key, const V& value, std::vector& path) { Page p; loadPage(page, p); // std::cout << "::VISIT " << p << std::endl; if (p.node_) { path.push_back(page); return insert(next(key, p), key, value, path); } LeafEntry* begin = p.leafPage().lentries_; LeafEntry* end = begin + p.count_; LeafEntry* e = std::lower_bound(begin, end, key); // std::cout << "::store " << key << " in " << p << std::endl; // std::cout << "insert at at " << (e - p.lentries_) << std::endl; if ((e != end) && ((*e).key_ == key)) { // std::cout << "Page " << p.id_ << " at pos " << e - begin << std::endl; // std::cout << "Replace " << key << std::endl << (*e).value_ << std::endl << value << // std::endl; (*e).value_ = value; savePage(p); return true; } // Assumes that K and V are PODs.... size_t count = p.count_ - (e - begin); // std::cout << "Move count " << count << std::endl; memmove(e + 1, e, count * sizeof(LeafEntry)); (*e).key_ = key; (*e).value_ = value; p.count_++; savePage(p); while ((p.node_ && (p.count_ == maxNodeEntries_)) || (!p.node_ && (p.count_ == maxLeafEntries_))) { // std::cout << p << " needs spliting" << std::endl; if (p.id_ == 1) { splitRoot(); return false; } else { int middle = p.count_ / 2; Page n; newPage(n); K k; // Same type n.node_ = p.node_; if (p.node_) { // std::cout << "SPLIT-NODE " << p << std::endl; for (size_t i = middle + 1; i < p.count_; ++i) { n.nodePage().nentries_[n.count_++] = p.nodePage().nentries_[i]; } ASSERT(n.count_ == p.count_ - middle - 1); k = p.nodePage().nentries_[middle].key_; p.count_ -= n.count_; p.count_--; n.left_ = p.nodePage().nentries_[middle].page_; // It's a node, Push-up } else { // std::cout << "SPLIT-LEAF " << p << std::endl; for (size_t i = middle; i < p.count_; ++i) { n.leafPage().lentries_[n.count_++] = p.leafPage().lentries_[i]; } ASSERT(n.count_ == p.count_ - middle); p.count_ -= n.count_; // It's a leaf, Copy-up k = n.leafPage().lentries_[0].key_; if (p.right_) { // TODO: do an I/O just for the linked list Page r; loadPage(p.right_, r); r.left_ = n.id_; savePage(r); } n.right_ = p.right_; n.left_ = p.id_; p.right_ = n.id_; } savePage(n); savePage(p); page = path.back(); path.pop_back(); loadPage(page, p); ASSERT(p.node_); NodeEntry* begin = p.nodePage().nentries_; NodeEntry* end = p.nodePage().nentries_ + p.count_; ASSERT(begin != end); NodeEntry* e = std::lower_bound(begin, end, k); // Node should not be there ASSERT(!(e != end && (*e).key_ == k)); size_t count = p.count_ - (e - begin); // std::cout << "Move node count " << count << std::endl; memmove(e + 1, e, count * sizeof(NodeEntry)); (*e).key_ = k; (*e).page_ = n.id_; p.count_++; savePage(p); } } return false; } template void BTree::splitRoot() { Page p; loadPage(1, p); Page pleft; Page pright; newPage(pleft); newPage(pright); // std::cout << "SPLIT ROOT " << p << std::endl; unsigned long middle = p.count_ / 2; K key; if (p.node_) { // dump(); pleft.node_ = true; pright.node_ = true; pleft.left_ = p.left_; for (unsigned long i = 0; i < middle; i++) { pleft.nodePage().nentries_[pleft.count_++] = p.nodePage().nentries_[i]; } pright.left_ = p.nodePage().nentries_[middle].page_; for (size_t i = middle + 1; i < p.count_; i++) { pright.nodePage().nentries_[pright.count_++] = p.nodePage().nentries_[i]; } key = p.nodePage().nentries_[middle].key_; } else { pleft.node_ = false; pright.node_ = false; for (unsigned long i = 0; i < middle; ++i) { // DEBUG_VAR( pleft.count_ ); pleft.leafPage().lentries_[pleft.count_++] = p.leafPage().lentries_[i]; } // Some version of Gcc (e.g. 4.8.1) optimize out the increment of this counter ASSERT(pleft.count_ == middle); for (size_t i = middle; i < p.count_; ++i) { pright.leafPage().lentries_[pright.count_++] = p.leafPage().lentries_[i]; } // Some version of Gcc (e.g. 4.8.1) optimize out the increment of this counter ASSERT(pright.count_ == p.count_ - middle); key = pright.leafPage().lentries_[0].key_; pleft.right_ = pright.id_; pright.left_ = pleft.id_; } zero(p); p.id_ = 1; p.count_ = 1; p.node_ = true; p.left_ = pleft.id_; p.nodePage().nentries_[0].key_ = key; p.nodePage().nentries_[0].page_ = pright.id_; savePage(pright); savePage(pleft); savePage(p); // cout << "LEFT\n" << pleft << endl; // cout << "RIGHT\n" << pright << endl; } template bool BTree::get(const K& key, V& value) { AutoSharedLock > lock(this); V result; if (search(1, key, result)) { // std::cout << "Found " << result << std::endl; value = result; return true; } // std::cout << "Not Found " << std::endl; return false; } template bool BTree::search(unsigned long page, const K& key, V& result) const { Page p; loadPage(page, p); // std::cout << "Search " << key << ", Visit " << p << std::endl; if (p.node_) { return search(next(key, p), key, result); } const LeafEntry* begin = p.leafPage().lentries_; const LeafEntry* end = begin + p.count_; const LeafEntry* e = std::lower_bound(begin, end, key); if ((e != end) && ((*e).key_ == key)) { result = (*e).value_; return true; } return false; } template template void BTree::range(const K& key1, const K& key2, T& result) { AutoSharedLock > lock(this); result.clear(); search(1, key1, key2, result); } template bool BTree::remove(const K&) { NOTIMP; } template template void BTree::search(unsigned long page, const K& key1, const K& key2, T& result) { Page p; loadPage(page, p); // std::cout << "Search " << key << ", Visit " << p << std::endl; if (p.node_) { return search(next(key1, p), key1, key2, result); } const LeafEntry* begin = p.leafPage().lentries_; const LeafEntry* end = begin + p.count_; const LeafEntry* e = std::lower_bound(begin, end, key1); if (e == end) return; // while((*e).key_ < key1) { e++; if (e == end) return; } // std::cout << "range " << (*e).key_ << " .... " << p.count_ << " " << (e - begin) << // std::endl; std::cout << " key1 " << key1 << std::endl; std::cout << " key2 " << key2 << // std::endl; // std::cout << " begin " << (*begin).key_ << std::endl; if (p.count_) { // unused const LeafEntry *last = begin + p.count_ -1; // std::cout << " last " << (*last).key_ << std::endl; } while (!(key2 < (*e).key_)) { // std::cout << "match " << p.id_ << " pos " << (e - begin) << " " << (*e).key_ << // std::endl; result.push_back(result_type((*e).key_, (*e).value_)); ++e; if (e == end) { if (p.right_) { loadPage(p.right_, p); ASSERT(!p.node_); e = p.leafPage().lentries_; end = e + p.count_; } else { return; } } } } template off_t BTree::pageOffset(unsigned long page) const { ASSERT(page > 0); // Root page is 1. 0 is leaf marker return sizeof(Page) * off_t(page - 1) + offset_; } template void BTree::_loadPage(unsigned long page, Page& p) const { // std::cout << "Load " << page << std::endl; off_t o = pageOffset(page); off_t here = file_.seek(o); ASSERT(here == o); int len = file_.read(&p, sizeof(p)); ASSERT(len == sizeof(p)); ASSERT(page == p.id_); } template void BTree::loadPage(unsigned long page, Page& p) const { BTree* self = const_cast*>(this); typename Cache::iterator j = self->cache_.find(page); if (j != self->cache_.end()) { // TODO: find someting better... memcpy(&p, (*j).second.page_, sizeof(Page)); return; } _loadPage(page, p); if (cacheReads_) { Page* q = new Page(); memcpy(q, &p, sizeof(Page)); self->cache_[p.id_] = _PageInfo(q); } } template void BTree::_savePage(const Page& p) { ASSERT(!readOnly_); // std::cout << "Save " << p << std::endl; off_t o = pageOffset(p.id_); off_t here; here = file_.seek(o); ASSERT(here == o); int len = file_.write(&p, sizeof(p)); ASSERT(len == sizeof(p)); } template void BTree::savePage(const Page& p) { BTree* self = const_cast*>(this); typename Cache::iterator j = self->cache_.find(p.id_); if (j != self->cache_.end()) { // TODO: find someting better... memcpy((*j).second.page_, &p, sizeof(Page)); (*j).second.dirty_ = true; (*j).second.count_++; return; } if (cacheWrites_) { Page* q = new Page(); memcpy(q, &p, sizeof(Page)); self->cache_[p.id_] = _PageInfo(q); j = self->cache_.find(p.id_); (*j).second.dirty_ = true; (*j).second.count_++; return; } _savePage(p); } template void BTree::_newPage(Page& p) { ASSERT(!readOnly_); off_t here = file_.seekEnd(); unsigned long long page = (here - offset_) / sizeof(Page) + 1; // std::cout << "NEWPAGE " << page << std::endl; zero(p); p.id_ = (unsigned long)page; ASSERT(page == p.id_); ASSERT(pageOffset(page) == here); int len = file_.write(&p, sizeof(p)); // TODO: a sparse file.... ASSERT(len == sizeof(p)); // return p.id_; } template void BTree::newPage(Page& p) { _newPage(p); if (cacheReads_ || cacheWrites_) { Page* q = new Page(); memcpy(q, &p, sizeof(Page)); cache_[p.id_] = _PageInfo(q); } } template size_t BTree::count() const { return count(1); } template size_t BTree::count(unsigned long page) const { AutoSharedLock > lock(const_cast(this)); Page p; loadPage(page, p); size_t c = 0; if (p.node_) { c += this->count(p.left_); for (unsigned long i = 0; i < p.count_; i++) c += count(p.nodePage().nentries_[i].page_); } else // leaf { c = p.count_; } return c; } template void BTree::lockShared() { L::lockRange(file_.fileno(), 0, 0, F_SETLKW, F_RDLCK); } template void BTree::lock() { L::lockRange(file_.fileno(), 0, 0, F_SETLKW, readOnly_ ? F_RDLCK : F_WRLCK); } template void BTree::unlock() { L::lockRange(file_.fileno(), 0, 0, F_SETLK, F_UNLCK); } template void BTree::flock() { if (::flock(file_.fileno(), readOnly_ ? LOCK_EX : LOCK_SH) < 0) { bool skip = false; #ifdef ENOTSUP if (errno == ENOTSUP) { skip = true; } #endif #ifdef ENOTSUPP if (errno == ENOTSUPP) { skip = true; } #endif #ifdef EOPNOTSUPP if (errno == EOPNOTSUPP) { skip = true; } #endif if (!skip) { std::stringstream ss; ss << "Error " << Log::syserr << " locking " << path_; throw eckit::UserError(ss.str(), Here()); } } } template void BTree::funlock() { if (::flock(file_.fileno(), LOCK_UN) < 0) { bool skip = false; #ifdef ENOTSUP if (errno == ENOTSUP) { skip = true; } #endif #ifdef ENOTSUPP if (errno == ENOTSUPP) { skip = true; } #endif #ifdef EOPNOTSUPP if (errno == EOPNOTSUPP) { skip = true; } #endif if (!skip) { std::stringstream ss; ss << "Error " << Log::syserr << " unlocking " << path_; throw eckit::UserError(ss.str(), Here()); } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/container/KDMapped.cc0000664000175000017500000000740315161702250020612 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "KDMapped.h" #include "eckit/filesystem/PathName.h" #include "eckit/os/Stat.h" #include #include #include #include #include // for memcpy #include "eckit/memory/MMap.h" namespace eckit { KDMapped::KDMapped(const PathName& path, size_t itemCount, size_t itemSize, size_t metadataSize) : path_(path), header_(itemCount, itemSize, metadataSize), size_(0), base_{nullptr}, root_(0), addr_{nullptr}, fd_(-1) { int oflags = O_RDWR | O_CREAT; int mflags = PROT_READ | PROT_WRITE; if (itemCount == 0) { oflags = O_RDWR; // mflags = PROT_READ; } size_t base; SYSCALL(fd_ = ::open(path.localPath(), oflags, 0777)); if (itemCount == 0) { readonly_ = true; Stat::Struct s; SYSCALL(Stat::stat(path.localPath(), &s)); size_ = s.st_size; int n; SYSCALL(n = ::read(fd_, &header_, sizeof(header_))); ASSERT(n == sizeof(header_)); lseek(fd_, 0, SEEK_SET); root_ = 1; ASSERT(header_.headerSize_ == sizeof(header_)); base = ((header_.headerSize_ + header_.metadataSize_ + header_.itemSize_ - 1) / header_.itemSize_) * header_.itemSize_; count_ = header_.itemCount_; } else { readonly_ = false; base = ((header_.headerSize_ + header_.metadataSize_ + header_.itemSize_ - 1) / header_.itemSize_) * header_.itemSize_; char c = 0; size_ = base + (itemCount + 1) * itemSize; lseek(fd_, 0, SEEK_SET); SYSCALL(::write(fd_, &header_, sizeof(header_))); lseek(fd_, size_ - 1, SEEK_SET); SYSCALL(::write(fd_, &c, 1)); } addr_ = MMap::mmap(nullptr, size_, mflags, MAP_SHARED, fd_, 0); if (addr_ == MAP_FAILED) { Log::error() << "open(" << path << ')' << Log::syserr << std::endl; throw FailedSystemCall("mmap"); } base_ = reinterpret_cast(addr_) + base; } KDMapped::~KDMapped() { if (addr_) { SYSCALL(munmap(addr_, size_)); } if (fd_ >= 0) { SYSCALL(close(fd_)); } } // Warning, takes ownership of maps KDMapped::KDMapped(const KDMapped& other) : path_(other.path_), header_(other.header_), count_(other.count_), size_(other.size_), base_(other.base_), root_(other.root_), addr_(other.addr_), fd_(other.fd_) { const_cast(other).addr_ = nullptr; const_cast(other).fd_ = -1; } // Warning, takes ownership of maps KDMapped& KDMapped::operator=(const KDMapped& other) { if (this == &other) { return *this; } path_ = other.path_; count_ = other.count_; size_ = other.size_; addr_ = other.addr_; fd_ = other.fd_; root_ = other.root_; header_ = other.header_; base_ = other.base_; const_cast(other).addr_ = nullptr; const_cast(other).fd_ = -1; return *this; } void KDMapped::setMetadata(const void* addr, size_t size) { ASSERT(size == header_.metadataSize_); char* start = static_cast(addr_); ::memcpy(start + sizeof(header_), addr, size); } void KDMapped::getMetadata(void* addr, size_t size) { ASSERT(size == header_.metadataSize_); char* start = static_cast(addr_); ::memcpy(addr, start + sizeof(header_), size); } } // namespace eckit eckit-2.0.7/src/eckit/container/CacheManager.cc0000664000175000017500000002425015161702250021462 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/container/CacheManager.h" #include #include #include #include #include #include "eckit/container/BTree.h" #include "eckit/log/Bytes.h" #include "eckit/runtime/Main.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- CacheManagerBase::CacheManagerBase(const std::string& loaderName, size_t maxCacheSize, const std::string& extension) : loaderName_(loaderName), maxCacheSize_(maxCacheSize), extension_(extension) {} CacheManagerBase::~CacheManagerBase() = default; std::string CacheManagerBase::loader() const { return loaderName_; } template static bool compare(const T& a, const T& b) { if (a.second.last_ == 0) { return false; } return a.second.last_ < b.second.last_; } static bool sub_path_of(const PathName& base, const PathName& path) { std::string subp = base.asString(); std::string p = path.asString(); // Log::debug() << "subp " << subp << std::endl; // Log::debug() << "p " << p << std::endl; std::size_t f = p.find(subp); return f == 0; } void CacheManagerBase::rescanCache(const PathName& base) const { ASSERT(btree_); PathName db(base / "cache-manager.btree"); PathName mapping(base / "cache-manager.mapping"); // already under lock from callee cache_btree_t& btree = *btree_; Log::info() << "CACHE-MANAGER cleanup " << db << ", rebuilding index" << std::endl; std::vector files; std::vector dirs; base.childrenRecursive(files, dirs); std::ofstream out(mapping.asString().c_str(), std::ios::app); for (const auto& file : files) { if (file.extension() != extension_) { continue; } Log::info() << "CACHE-MANAGER cleanup " << db << ", indexing " << file << std::endl; MD5 md5(file); out << md5.digest() << " " << file << std::endl; cache_key_t key(md5.digest()); cache_entry_t entry = {0, 0, 0}; cache_key_t total_key(".total"); cache_entry_t total_entry = {0, 0, 0}; if (!btree.get(key, entry)) { entry.size_ = size_t(file.size()); entry.count_ = 1; entry.last_ = ::time(nullptr); btree.set(key, entry); if (btree.get(total_key, total_entry)) { total_entry.size_ += entry.size_; total_entry.count_++; } btree.set(total_key, total_entry); } } } void CacheManagerBase::touch(const PathName& base, const PathName& path) const { // 1- Do we do it (bool) // 2- where do we store the data (path) // 3- What is the threshold (size_t) if (!maxCacheSize_) { return; } if (not writable(base)) { Log::warning() << "CACHE-MANAGER base " << base << " isn't writable, cannot update cache management" << std::endl; } AutoUmask umask(0); PathName db(base / "cache-manager.btree"); PathName mapping(base / "cache-manager.mapping"); try { if (!btree_) { btree_.reset(new cache_btree_t(db)); } cache_btree_t& btree = *btree_; MD5 md5(path); cache_key_t key(md5.digest()); cache_entry_t entry = {0, 0, 0}; if (!btree.get(key, entry)) { // entry not in cache-manager.btree FileLock lock(mapping); AutoLock locker(lock); if (mapping.size() == Length(0)) { rescanCache(base); return; // entry that was missing got also inserted with full rescan } cache_key_t total_key(".total"); cache_entry_t total_entry = {0, 0, 0}; entry.size_ = size_t(path.size()); if (btree.get(total_key, total_entry)) { total_entry.size_ += entry.size_; total_entry.count_++; } btree.set(total_key, total_entry); { std::ofstream out(mapping.asString().c_str(), std::ios::app); out << md5.digest() << " " << path << std::endl; } if (total_entry.size_ > maxCacheSize_) { size_t remove = total_entry.size_ - maxCacheSize_; // Cleanup Log::info() << "CACHE-MANAGER cleanup " << db << ", size is " << Bytes(total_entry.size_) << ", max size is " << Bytes(maxCacheSize_) << ", removing " << Bytes(remove) << std::endl; std::map md5_to_path; std::ifstream in(mapping.asString().c_str()); std::string s, t; bool rescan = false; while (in >> s >> t) { PathName p(t); if (s.length() != MD5_DIGEST_LENGTH * 2 || not sub_path_of(base, p)) { Log::warning() << "CACHE-MANAGER cleanup " << mapping << ", invalid entry [" << s << "] and [" << t << "], ignoring but will rebuild index later" << std::endl; rescan = true; continue; } md5_to_path[s] = p; } std::deque result; cache_key_t first; memset(first.data(), '0', cache_key_t::static_size()); cache_key_t last; memset(last.data(), 'f', cache_key_t::static_size()); btree.range(first, last, result); std::sort(result.begin(), result.end(), compare); size_t deleted = 0; while (deleted < remove && !result.empty()) { cache_btree_t::result_type p = result.front(); result.pop_front(); if (p.second.last_ == 0) { // entry was cleared, its important to skip else below we retrigger scan continue; } std::map::iterator j = md5_to_path.find(p.first); if (j == md5_to_path.end()) { // in the btree, but not file mapping. Path may exist on disk rescan = true; continue; } const PathName& file = (*j).second; size_t unlinked = 0; if (file.exists()) { file.unlink(); unlinked = p.second.size_; PathName(file + ".lock").unlink(); } cache_entry_t zero = {0, 0, 0}; total_entry.size_ -= std::min(p.second.size_, total_entry.size_); total_entry.count_ -= std::min(size_t(1), total_entry.count_); btree.set(p.first, zero); btree.set(total_key, total_entry); deleted += unlinked; Log::warning() << "CACHE-MANAGER cleanup " << file << ", deleted: " << Bytes(unlinked) << std::endl; } if (deleted < remove) { Log::warning() << "CACHE-MANAGER cleanup " << mapping << ", could not delete enough space" << " total is " << Bytes(total_entry.size_) << std::endl; } if (rescan) { Log::warning() << "CACHE-MANAGER cleanup " << mapping << ", inconsitent cache index, needs rebuilding" << std::endl; // TODO: rebuild index + mapping from directory scan mapping.unlink(); mapping.touch(); rescanCache(base); } } } entry.last_ = ::time(nullptr); entry.count_++; btree.set(key, entry); Log::debug() << "CACHE-MANAGER touched path " << path << std::endl; } catch (std::exception& e) { Log::error() << "Error updating " << db << ", turning off" << Log::syserr << std::endl; const_cast(this)->maxCacheSize_ = 0; } } bool CacheManagerBase::writable(const PathName& path) const { return (::access(path.asString().c_str(), W_OK) == 0); } //---------------------------------------------------------------------------------------------------------------------- static PathName lockFile(const std::string& path) { AutoUmask umask(0); PathName lock(path + ".lock"); lock.touch(); return lock; } CacheManagerFileSemaphoreLock::CacheManagerFileSemaphoreLock(const std::string& path) : path_(lockFile(path)), lock_(path_) {} void CacheManagerFileSemaphoreLock::lock() { AutoUmask umask(0); Log::info() << "Wait for lock " << path_ << std::endl; lock_.lock(); Log::info() << "Got lock " << path_ << std::endl; std::string hostname = Main::hostname(); std::ofstream os(path_.asString().c_str()); os << hostname << " " << ::getpid() << std::endl; } void CacheManagerFileSemaphoreLock::unlock() { AutoUmask umask(0); Log::info() << "Unlock " << path_ << std::endl; std::ofstream os(path_.asString().c_str()); os << std::endl; lock_.unlock(); } //---------------------------------------------------------------------------------------------------------------------- CacheManagerFileFlock::CacheManagerFileFlock(const std::string& path) : lock_(lockFile(path), true /*unlink lock at destruction*/) {} void CacheManagerFileFlock::lock() { lock_.lock(); } void CacheManagerFileFlock::unlock() { lock_.unlock(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/container/ClassExtent.h0000664000175000017500000001606415161702250021267 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File ClassExtent.h // Baudouin Raoult - ECMWF Jul 96 #ifndef eckit_ClassExtent_h #define eckit_ClassExtent_h #include #include "eckit/exception/Exceptions.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" namespace eckit { //----------------------------------------------------------------------------- template class ClassExtent { public: // -- Contructors ClassExtent(T*); ClassExtent(const ClassExtent&) = delete; ClassExtent& operator=(const ClassExtent&) = delete; ClassExtent(ClassExtent&&) = delete; ClassExtent& operator=(ClassExtent&&) = delete; // -- Destructor ~ClassExtent(); // -- Methods static size_t size(); public: // methods static void callAll(void (T::*)()); static void callAll(void (T::*)() const); template static void callAll(void (T::*)(P), P); template static void callAll(void (T::*)(P) const, P); template static void callAll(void (T::*)(P1, P2), P1, P2); template static void callAll(void (T::*)(P&) const, P&); template static void callAll(void (T::*)(P&), P&); template static void callAll(void (T::*)(P1&, P2&), P1&, P2&); private: // members struct Extent { using Map = std::map*, T*, std::less*>>; Mutex mutex_; Map map_; bool inited_; Extent(); ~Extent(); }; static Extent extent_; }; //----------------------------------------------------------------------------- // We assume that global-initialisation is single threaded template typename ClassExtent::Extent ClassExtent::extent_; template ClassExtent::ClassExtent(T* obj) { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); extent_.map_[this] = obj; } template ClassExtent::~ClassExtent() { if (extent_.inited_) // This can be after exit() is called // I need to find a solution { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); ASSERT(extent_.map_.find(this) != extent_.map_.end()); extent_.map_.erase(this); } } template size_t ClassExtent::size() { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); return extent_.map_.size(); } template void ClassExtent::callAll(void (T::*proc)()) { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); // Make a copy to cater for object that are deleted during the loop typename ClassExtent::Extent::Map map = extent_.map_; // for(ClassExtent::Extent::Map::iterator i = extent_.map_.begin(); using map_type = typename ClassExtent::Extent::Map; typename map_type::iterator i; for (i = map.begin(); i != map.end(); ++i) { ((*i).second->*proc)(); } } template void ClassExtent::callAll(void (T::*proc)() const) { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); // for(ClassExtent::Extent::Map::iterator i = extent_.map_.begin(); using map_type = typename ClassExtent::Extent::Map; typename map_type::iterator i; for (i = extent_.map_.begin(); i != extent_.map_.end(); ++i) { ((*i).second->*proc)(); } } template template void ClassExtent::callAll(void (T::*proc)(P), P arg) { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); // Make a copy to cater for object that are deleted during the loop typename ClassExtent::Extent::Map map = extent_.map_; // for(ClassExtent::Extent::Map::iterator i = extent_.map_.begin(); using map_type = typename ClassExtent::Extent::Map; typename map_type::iterator i; for (i = map.begin(); i != map.end(); ++i) { ((*i).second->*proc)(arg); } } template template void ClassExtent::callAll(void (T::*proc)(P) const, P arg) { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); // for(ClassExtent::Extent::Map::iterator i = extent_.map_.begin(); using map_type = typename ClassExtent::Extent::Map; typename map_type::iterator i; for (i = extent_.map_.begin(); i != extent_.map_.end(); ++i) { ((*i).second->*proc)(arg); } } template template void ClassExtent::callAll(void (T::*proc)(P1, P2), P1 arg1, P2 arg2) { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); // Make a copy to cater for object that are deleted during the loop typename ClassExtent::Extent::Map map = extent_.map_; // for(ClassExtent::Extent::Map::iterator i = extent_.map_.begin(); using map_type = typename ClassExtent::Extent::Map; typename map_type::iterator i; for (i = map.begin(); i != map.end(); ++i) { ((*i).second->*proc)(arg1, arg2); } } template template void ClassExtent::callAll(void (T::*proc)(P&), P& arg) { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); // for(ClassExtent::Extent::Map::iterator i = extent_.map_.begin(); using map_type = typename ClassExtent::Extent::Map; typename map_type::iterator i; for (i = extent_.map_.begin(); i != extent_.map_.end(); ++i) { ((*i).second->*proc)(arg); } } template template void ClassExtent::callAll(void (T::*proc)(P&) const, P& arg) { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); // for(ClassExtent::Extent::Map::iterator i = extent_.map_.begin(); using map_type = typename ClassExtent::Extent::Map; typename map_type::iterator i; for (i = extent_.map_.begin(); i != extent_.map_.end(); ++i) { ((*i).second->*proc)(arg); } } template template void ClassExtent::callAll(void (T::*proc)(P1&, P2&), P1& arg1, P2& arg2) { ASSERT(extent_.inited_); AutoLock lock(extent_.mutex_); // Make a copy to cater for object that are deleted during the loop typename ClassExtent::Extent::Map map = extent_.map_; // for(ClassExtent::Extent::Map::iterator i = extent_.map_.begin(); using map_type = typename ClassExtent::Extent::Map; typename map_type::iterator i; for (i = map.begin(); i != map.end(); ++i) { ((*i).second->*proc)(arg1, arg2); } } template ClassExtent::Extent::Extent() : inited_(true) {} template ClassExtent::Extent::~Extent() { inited_ = false; } //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/container/MappedArray.h0000664000175000017500000000452715161702250021240 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File MappedArray.h // Baudouin Raoult - ECMWF Nov 96 #ifndef eckit_MappedArray_h #define eckit_MappedArray_h #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/os/Semaphore.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- // Used to std::map an array to a file template class MappedArray { public: // stl compatibility using iterator = T*; using const_iterator = const T*; // -- Contructors MappedArray(const PathName&, unsigned long); MappedArray(const MappedArray&) = delete; MappedArray& operator=(const MappedArray&) = delete; // -- Destructor ~MappedArray(); // -- Methods void sync(); void lock() { sem_.lock(); } void unlock() { sem_.unlock(); } // stl compatibility iterator begin() { return array_; } iterator end() { return array_ + size_; } const_iterator begin() const { return array_; } const_iterator end() const { return array_ + size_; } unsigned long size() { return size_; } T& operator[](unsigned long n) { return array_[n]; } private: // members Semaphore sem_; void* map_; int fd_; T* array_; unsigned long size_; static unsigned long mapped_array_version() { return 1; } struct Header { uint32_t version_; uint32_t headerSize_; uint32_t elemSize_; Header() : version_(mapped_array_version()), headerSize_(sizeof(Header)), elemSize_(sizeof(T)) {} void validate() { ASSERT(version_ == mapped_array_version()); ASSERT(headerSize_ == sizeof(Header)); ASSERT(elemSize_ == sizeof(T)); } }; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #include "MappedArray.cc" #endif eckit-2.0.7/src/eckit/container/KDMemory.h0000664000175000017500000000416415161702250020517 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef KDMemory_H #define KDMemory_H #include "eckit/eckit.h" #include #include #include "eckit/container/StatCollector.h" //------------------------------------------------------------------------------------------------------ namespace eckit { //------------------------------------------------------------------------------------------------------ struct KDMemory : public StatCollector { using Ptr = void*; Ptr root() const { return nullptr; } void root(Ptr) {} template Ptr convert(Node* p) { return p; } template Node* convert(Ptr p, const Node*) { return static_cast(p); } template Node* newNode1(const A& a, const Node*) { nbItems_++; return new Node(a); } template Node* newNode2(const A& a, const B& b, const Node*) { nbItems_++; return new Node(a, b); } template Node* newNode3(const A& a, const B& b, const C& c, const Node*) { nbItems_++; return new Node(a, b, c); } template void deleteNode(Ptr p, const Node*) { Node* n = static_cast(p); if (n) { deleteNode(n->left(*this), n); deleteNode(n->right(*this), n); delete n; nbItems_--; } } size_t nbItems() const { return nbItems_; } private: size_t nbItems_{0}; }; template struct TT : public T { using Alloc = A; }; //------------------------------------------------------------------------------------------------------ } // end namespace eckit #endif eckit-2.0.7/src/eckit/container/SharedMemArray.cc0000664000175000017500000000533615161702250022034 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template SharedMemArray::SharedMemArray(const PathName& path, const std::string& shmName, size_t size) : sem_(path), size_(size), shmName_(shmName) { eckit::Log::debug() << "SharedMemArray semaphore path=" << path << ", size=" << size << ", shmName=" << shmName << std::endl; AutoLock lock(sem_); using PaddedHeader = Padded::Header, 4096>; fd_ = ::shm_open(shmName_.c_str(), O_RDWR | O_CREAT, 0777); if (fd_ < 0) { Log::error() << "shm_open(" << shmName_ << ')' << Log::syserr << std::endl; throw FailedSystemCall("shm_open", Here()); } Stat::Struct s; SYSCALL(Stat::fstat(fd_, &s)); off_t length = size_ * sizeof(T) + sizeof(PaddedHeader); eckit::Log::debug() << "SharedMemArray fd_=" << fd_ << ", s.st_size=" << s.st_size << ", length=" << length << std::endl; // Resize if needed bool zero = false; if (length > s.st_size) { SYSCALL(::ftruncate(fd_, length)); zero = true; } map_ = MMap::mmap(0, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0); if (map_ == MAP_FAILED) { Log::error() << "SharedMemArray name=" << shmName_ << " size=" << size << " fails to mmap(0,length,PROT_READ|PROT_WRITE,MAP_SHARED,fd_,0)" << Log::syserr << std::endl; throw FailedSystemCall("mmap", Here()); } if (zero) { ::memset(map_, 0, sizeof(PaddedHeader) + size_ * sizeof(T)); new (map_) PaddedHeader(); } else { ((PaddedHeader*)map_)->validate(); } array_ = (T*)(((char*)map_) + sizeof(PaddedHeader)); } template SharedMemArray::~SharedMemArray() { // Unmap here... } template void SharedMemArray::sync() { // int ret = fsync(fd_); // while(ret < 0 && errno == EINTR) // ret = fsync(fd_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/container/Trie.h0000664000175000017500000000364215161702250017733 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file trie.h /// @author Baudouin Raoult /// @author Simon Smart /// @date March 2017 #ifndef eckit_containers_Trie_H #define eckit_containers_Trie_H #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template class Trie { public: // methods Trie(); Trie(const Trie&) = delete; Trie& operator=(const Trie&) = delete; Trie(Trie&&) = delete; Trie& operator=(Trie&&) = delete; ~Trie(); bool empty() const { return kids_.empty() && !set_; } void insert(const std::string& key, T value); void remove(const std::string& key); bool contains(const std::string& key) const; T* find(const std::string& key) const; protected: // methods void print(std::ostream&) const; private: // members Trie* find(const unsigned char*, bool); bool remove(const unsigned char* key); private: // members unsigned short from_; /// Do we have a value in the value_ (or is just uninintialised POD/default constructed). bool set_; std::vector*> kids_; T value_; private: // friends friend std::ostream& operator<<(std::ostream& s, const Trie& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #include "Trie.cc" #endif // eckit_containers_Trie_H eckit-2.0.7/src/eckit/container/MappedArray.cc0000664000175000017500000000543615161702250021376 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit.h" #include #include #include #include #include #include #include #include "eckit/container/MappedArray.h" #include "eckit/memory/MMap.h" #include "eckit/memory/Padded.h" #include "eckit/os/Stat.h" #include "eckit/thread/AutoLock.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template MappedArray::MappedArray(const PathName& path, unsigned long size) : sem_(path), size_(size) { AutoLock lock(sem_); using PaddedHeader = Padded::Header, 4096>; fd_ = ::open(path.localPath(), O_RDWR | O_CREAT, 0777); if (fd_ < 0) { Log::error() << "open(" << path << ')' << Log::syserr << std::endl; throw FailedSystemCall("open", Here()); } Stat::Struct s; SYSCALL(Stat::stat(path.localPath(), &s)); bool initHeader = s.st_size < static_cast(sizeof(PaddedHeader)); off_t length = size_ * sizeof(T) + sizeof(PaddedHeader); // Resize if needed if (s.st_size != length) { SYSCALL(::ftruncate(fd_, length)); char buf1[sizeof(PaddedHeader)]; ::memset(buf1, 0, sizeof(buf1)); char buf2[sizeof(T)]; ::memset(buf2, 0, sizeof(buf2)); SYSCALL(write(fd_, buf1, sizeof(buf1))); for (size_t i = 0; i < size_; i++) SYSCALL(write(fd_, buf2, sizeof(buf2))); } map_ = MMap::mmap(0, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0); if (map_ == MAP_FAILED) { Log::error() << "MappedArray path=" << path << " size=" << size << " fails to mmap(0,length,PROT_READ|PROT_WRITE,MAP_SHARED,fd_,0)" << Log::syserr << std::endl; throw FailedSystemCall("mmap", Here()); } // If first time in, init header if (initHeader) new (map_) PaddedHeader(); else ((PaddedHeader*)map_)->validate(); array_ = (T*)(((char*)map_) + sizeof(PaddedHeader)); } template MappedArray::~MappedArray() { // Unmap here... } template void MappedArray::sync() { int ret = fsync(fd_); while (ret < 0 && errno == EINTR) ret = fsync(fd_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/container/KDMapped.h0000664000175000017500000000622315161702250020453 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef KDMapped_H #define KDMapped_H #include "eckit/container/StatCollector.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" namespace eckit { struct KDMappedHeader { size_t headerSize_; size_t itemCount_; size_t itemSize_; size_t metadataSize_; KDMappedHeader(size_t itemCount, size_t itemSize, size_t metadataSize) : headerSize_(sizeof(KDMappedHeader)), itemCount_(itemCount), itemSize_(itemSize), metadataSize_(metadataSize) {} }; class KDMapped : public StatCollector { public: KDMapped(const PathName&, size_t itemCount, size_t itemSize, size_t metadataSize); ~KDMapped(); KDMapped(const KDMapped& other); KDMapped& operator=(const KDMapped& other); using Ptr = size_t; template Node* base(const Node*) { ASSERT(sizeof(Node) == header_.itemSize_); return reinterpret_cast(base_); } Ptr root() const { return root_; } void root(Ptr r) { ASSERT(r == 1); } template Ptr convert(Node* p) { return p ? p - base(p) : 0; } template Node* convert(Ptr p, const Node* dummy) { Node* r = base(dummy); /* ASSERT(p < count_); */ return p ? &r[p] : nullptr; } template Node* newNode1(const A& a, const Node* dummy) { Node* r = base(dummy); ASSERT(!readonly_); // ASSERT(count_ * sizeof(Node) < size_); return new (&r[++count_]) Node(a); } template Node* newNode2(const A& a, const B& b, const Node* dummy) { Node* r = base(dummy); ASSERT(!readonly_); // ASSERT(count_ * sizeof(Node) < size_); return new (&r[++count_]) Node(a, b); } template Node* newNode3(const A& a, const B& b, const C& c, const Node* dummy) { Node* r = base(dummy); ASSERT(!readonly_); // ASSERT(count_ * sizeof(Node) < size_); return new (&r[++count_]) Node(a, b, c); } template void deleteNode(Ptr p, Node* n) { // Ignore // TODO: recycle space if needed } void setMetadata(const void*, size_t); void getMetadata(void*, size_t); template void setMetadata(const Metadata& meta) { setMetadata(&meta, sizeof(meta)); } template void getMetadata(Metadata& meta) { getMetadata(&meta, sizeof(meta)); } size_t nbItems() const { return count_; } private: PathName path_; KDMappedHeader header_; size_t count_{0}; bool readonly_{true}; long long size_; char* base_; Ptr root_; void* addr_; int fd_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/container/BloomFilter.h0000664000175000017500000000400015161702250021233 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file trie.h /// @author Baudouin Raoult /// @author Simon Smart /// @date March 2017 #ifndef eckit_containers_BloomFilter_H #define eckit_containers_BloomFilter_H #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template class BloomFilter { public: // types using data_type = unsigned long long; public: // methods BloomFilter(size_t size); BloomFilter(const BloomFilter&) = delete; BloomFilter& operator=(const BloomFilter&) = delete; BloomFilter(BloomFilter&&) = delete; BloomFilter& operator=(BloomFilter&&) = delete; ~BloomFilter(); bool empty() const; void insert(const T& value); bool contains(const T& value) const; protected: // methods void print(std::ostream&) const; private: // members static size_t elementCount(size_t nbits); /// Which bit should we be considering? size_t index(const T& value) const; private: // members size_t size_; size_t entries_; // n.b. We don't use std::vector, as it doesn't behave like a std::vector. // Better to be explicit std::vector data_; private: // friends friend std::ostream& operator<<(std::ostream& s, const BloomFilter& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #include "BloomFilter.cc" #endif // eckit_containers_BloomFilter_H eckit-2.0.7/src/eckit/container/StatCollector.h0000664000175000017500000000514515161702250021612 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File StatCollector.h // Baudouin Raoult - ECMWF Apr 97 #ifndef eckit_StatCollector_h #define eckit_StatCollector_h #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/BigNum.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- struct StatCollector { StatCollector() { statsReset(); depth_ = 0; } // -- Methods void statsCall() { calls_++; } void statsVisitNode() { nodes_++; } void statsDepth(size_t d) { if (d > depth_) { depth_ = d; } } void statsNewCandidateOK() { newCandidateOK_++; } void statsNewCandidateMiss() { newCandidateMiss_++; } void statsCrossOver() { crossOvers_++; } void statsReset() { crossOvers_ = calls_ = newCandidateOK_ = newCandidateMiss_ = nodes_ = 0; } void print(std::ostream& s) const { s << "Stats calls: " << BigNum(calls_) << " avg candidates: " << BigNum(double(newCandidateMiss_ + newCandidateOK_) / double(calls_) + 0.5) << ", avg nodes: " << BigNum(double(nodes_) / double(calls_) + 0.5) << ", depth: " << depth_; } void statsPrint(std::ostream& s, bool fancy) const { if (fancy) { s << *this << std::endl; } else { s << " calls: " << BigNum(calls_) << std::endl; s << " miss: " << BigNum(newCandidateMiss_) << std::endl; s << " hit: " << BigNum(newCandidateOK_) << std::endl; s << " nodes: " << BigNum(nodes_) << std::endl; s << " depth: " << BigNum(depth_) << std::endl; s << " crossovers: " << BigNum(crossOvers_) << std::endl; } } // -- Members size_t calls_; size_t nodes_; size_t depth_; size_t newCandidateMiss_; size_t newCandidateOK_; size_t crossOvers_; // -- Friends friend std::ostream& operator<<(std::ostream& s, const StatCollector& p) { p.print(s); return s; } }; //----------------------------------------------------------------------------- // } // namespace eckit #endif eckit-2.0.7/src/eckit/container/SharedMemArray.h0000664000175000017500000000514315161702250021672 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date December 2016 #ifndef eckit_SharedMemArray_h #define eckit_SharedMemArray_h #include #include "eckit/os/Semaphore.h" #include "eckit/memory/Padded.h" #include "eckit/thread/AutoLock.h" #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/memory/MMap.h" #include "eckit/os/Stat.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// Maps an array to shared memory template class SharedMemArray { public: // types using iterator = T*; using const_iterator = const T*; public: // methods SharedMemArray(const PathName&, const std::string& shmName, size_t); SharedMemArray(const SharedMemArray&) = delete; SharedMemArray& operator=(const SharedMemArray&) = delete; SharedMemArray(SharedMemArray&&) = delete; SharedMemArray& operator=(SharedMemArray&&) = delete; ~SharedMemArray(); void sync(); void lock() { sem_.lock(); } void unlock() { sem_.unlock(); } iterator begin() { return array_; } iterator end() { return array_ + size_; } const_iterator begin() const { return array_; } const_iterator end() const { return array_ + size_; } unsigned long size() { return size_; } T& operator[](unsigned long n) { return array_[n]; } private: // members Semaphore sem_; void* map_; int fd_; T* array_; size_t size_; std::string shmName_; static unsigned long shared_mem_array_version() { return 1; } struct Header { uint32_t version_; uint32_t headerSize_; uint32_t elemSize_; Header() : version_(shared_mem_array_version()), headerSize_(sizeof(Header)), elemSize_(sizeof(T)) {} void validate() { ASSERT(version_ == shared_mem_array_version()); ASSERT(headerSize_ == sizeof(Header)); ASSERT(elemSize_ == sizeof(T)); } }; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #include "SharedMemArray.cc" #endif eckit-2.0.7/src/eckit/message/0000775000175000017500000000000015161702250016314 5ustar alastairalastaireckit-2.0.7/src/eckit/message/Decoder.cc0000664000175000017500000000410315161702250020166 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/message/Decoder.h" #include "eckit/exception/Exceptions.h" #include "eckit/message/Message.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" #include #include #include namespace eckit::message { //---------------------------------------------------------------------------------------------------------------------- namespace { eckit::Mutex* local_mutex = nullptr; std::vector* decoders = nullptr; pthread_once_t once = PTHREAD_ONCE_INIT; void init() { local_mutex = new eckit::Mutex(); decoders = new std::vector(); } size_t index = 0; } // namespace MessageDecoder::MessageDecoder() { pthread_once(&once, init); eckit::AutoLock lock(local_mutex); decoders->push_back(this); } MessageDecoder::~MessageDecoder() { eckit::AutoLock lock(local_mutex); decoders->erase(std::remove(decoders->begin(), decoders->end(), this), decoders->end()); } MessageDecoder& MessageDecoder::lookup(const Message& msg) { eckit::AutoLock lock(local_mutex); size_t n = decoders->size(); ASSERT(n); for (size_t i = 0; i < n; ++i) { MessageDecoder* d = (*decoders)[(i + index) % n]; if (d->match(msg)) { index = i; // Start with this index for next message return *d; } } std::ostringstream oss; oss << "Cannot find a Decoder for " << msg << std::endl; throw eckit::SeriousBug(oss.str()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::message eckit-2.0.7/src/eckit/message/MessageContent.cc0000664000175000017500000000732615161702250021552 0ustar alastairalastair/* * (C) Copyright 2017- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Jun 2020 #include "eckit/memory/Counted.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/DataHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/message/Decoder.h" #include "eckit/message/MessageContent.h" namespace eckit::message { MessageContent::operator bool() const { return true; } void MessageContent::write(eckit::DataHandle&) const { std::ostringstream oss; oss << "Not implemented " << *this << " write()"; throw eckit::SeriousBug(oss.str()); } size_t MessageContent::length() const { std::ostringstream oss; oss << "Not implemented " << *this << " length(get)"; throw eckit::SeriousBug(oss.str()); } std::string MessageContent::getString(const std::string& key) const { std::ostringstream oss; oss << "Not implemented " << *this << " getString()"; throw eckit::SeriousBug(oss.str()); } long MessageContent::getLong(const std::string& key) const { std::ostringstream oss; oss << "Not implemented " << *this << " getLong()"; throw eckit::SeriousBug(oss.str()); } double MessageContent::getDouble(const std::string& key) const { std::ostringstream oss; oss << "Not implemented " << *this << " getDouble()"; throw eckit::SeriousBug(oss.str()); } void MessageContent::getDoubleArray(const std::string& key, std::vector&) const { std::ostringstream oss; oss << "Not implemented " << *this << " getDoubleArray(key, vector&)"; throw eckit::SeriousBug(oss.str()); } void MessageContent::getFloatArray(const std::string& key, std::vector&) const { std::ostringstream oss; oss << "Not implemented " << *this << " getFloatArray(key, vector&)"; throw eckit::SeriousBug(oss.str()); } void MessageContent::getDoubleArray(const std::string& key, double* data, size_t len) const { std::ostringstream oss; oss << "Not implemented " << *this << " getDoubleArray(key, double*, len)"; throw eckit::SeriousBug(oss.str()); } void MessageContent::getFloatArray(const std::string& key, float* data, size_t len) const { std::ostringstream oss; oss << "Not implemented " << *this << " getFloatArray(key, float*, len)"; throw eckit::SeriousBug(oss.str()); } size_t MessageContent::getSize(const std::string& key) const { std::ostringstream oss; oss << "Not implemented " << *this << " getSize(key)"; throw eckit::SeriousBug(oss.str()); } eckit::DataHandle* MessageContent::readHandle() const { std::ostringstream oss; oss << "Not implemented " << *this << " readHandle()"; throw eckit::SeriousBug(oss.str()); } eckit::Offset MessageContent::offset() const { std::ostringstream oss; oss << "Not implemented " << *this << " offset()"; throw eckit::SeriousBug(oss.str()); } const void* MessageContent::data() const { std::ostringstream oss; oss << "Not implemented " << *this << " data()"; throw eckit::SeriousBug(oss.str()); } void MessageContent::transform(const Transformer&) { std::ostringstream oss; oss << "Not implemented " << *this << " tranform(Transformer)"; throw eckit::SeriousBug(oss.str()); } void MessageContent::transform(const OrderedStringDict&) { std::ostringstream oss; oss << "Not implemented " << *this << " tranform(OrderedStringDict)"; throw eckit::SeriousBug(oss.str()); } } // namespace eckit::message eckit-2.0.7/src/eckit/message/Message.h0000664000175000017500000001014615161702250020053 0ustar alastairalastair/* * (C) Copyright 2017- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Jun 2020 #ifndef metkit_data_Message_H #define metkit_data_Message_H #include #include #include "eckit/io/Buffer.h" #include "eckit/message/Decoder.h" #include "eckit/types/Types.h" namespace eckit { class DataHandle; class Offset; class PathName; }; // namespace eckit namespace eckit { namespace mars { class MarsRequest; } namespace message { class MessageContent; class CodesContent; //---------------------------------------------------------------------------------------------------------------------- class MetadataGatherer { public: virtual ~MetadataGatherer(); virtual void setValue(const std::string& key, const std::string& value) = 0; virtual void setValue(const std::string& key, long value) = 0; virtual void setValue(const std::string& key, double value) = 0; }; //---------------------------------------------------------------------------------------------------------------------- /// Message represents a data object with metadata attached /// Applying a transformation to a message modifies it in place class Message { public: Message(); explicit Message(MessageContent*); Message(const Message&); ~Message(); Message& operator=(const Message&); explicit operator bool() const; void write(eckit::DataHandle&) const; size_t length() const; eckit::Offset offset() const; const void* data() const; std::string getString(const std::string& key) const; long getLong(const std::string& key) const; double getDouble(const std::string& key) const; void getDoubleArray(const std::string& key, std::vector&) const; void getFloatArray(const std::string& key, std::vector&) const; size_t getSize(const std::string& key) const; // Write double array at key to pre allocated array void getDoubleArray(const std::string& key, double* data, size_t len) const; void getFloatArray(const std::string& key, float* data, size_t len) const; void getMetadata(MetadataGatherer&, GetMetadataOptions options = GetMetadataOptions{}) const; eckit::Buffer decode() const; eckit::DataHandle* readHandle() const; mars::MarsRequest request() const; void transform(const eckit::OrderedStringDict& modifiers); private: MessageContent* content_; mutable MessageDecoder* decoder_ = nullptr; // non-owning MessageDecoder& lookupDecoder() const; void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const Message& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- template class StringSetter : public MetadataGatherer { T& object_; void setValue(const std::string& key, const std::string& value) override { object_.setValue(key, value); } void setValue(const std::string& /*key*/, long /*value*/) override {} void setValue(const std::string& /*key*/, double /*value*/) override {} public: StringSetter(T& object) : object_(object) {} }; template class TypedSetter : public MetadataGatherer { T& object_; void setValue(const std::string& key, const std::string& value) override { object_.setValue(key, value); } void setValue(const std::string& key, long value) override { object_.setValue(key, value); } void setValue(const std::string& key, double value) override { object_.setValue(key, value); } public: TypedSetter(T& object) : object_(object) {} }; //---------------------------------------------------------------------------------------------------------------------- } // namespace message } // namespace eckit #endif eckit-2.0.7/src/eckit/message/Message.cc0000664000175000017500000000775215161702250020222 0ustar alastairalastair/* * (C) Copyright 2017- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Jun 2020 #include #include "eckit/io/Offset.h" #include "eckit/message/Decoder.h" #include "eckit/message/Message.h" #include "eckit/message/MessageContent.h" #include "eckit/types/Types.h" namespace eckit::message { //---------------------------------------------------------------------------------------------------------------------- class NoContent : public MessageContent { virtual operator bool() const { return false; } void print(std::ostream& s) const { s << "NoContent[]"; } void* operator new(size_t); public: NoContent() { attach(); } }; static NoContent noContent; //---------------------------------------------------------------------------------------------------------------------- Message::Message() : content_(&noContent) { content_->attach(); } Message::Message(MessageContent* content) : content_(content ? content : &noContent) { content_->attach(); } Message::Message(const Message& other) : content_(other.content_) { content_->attach(); } Message& Message::operator=(const Message& other) { if (this == &other) { return *this; } if (content_ != other.content_) { content_->detach(); content_ = other.content_; content_->attach(); } return *this; } Message::~Message() { content_->detach(); } void Message::print(std::ostream& s) const { s << "Message[" << *content_ << "]"; } Message::operator bool() const { return content_->operator bool(); } void Message::write(eckit::DataHandle& handle) const { content_->write(handle); } size_t Message::length() const { return content_->length(); } std::string Message::getString(const std::string& key) const { return content_->getString(key); } long Message::getLong(const std::string& key) const { return content_->getLong(key); } double Message::getDouble(const std::string& key) const { return content_->getDouble(key); } void Message::getDoubleArray(const std::string& key, std::vector& v) const { return content_->getDoubleArray(key, v); } void Message::getFloatArray(const std::string& key, std::vector& v) const { return content_->getFloatArray(key, v); } size_t Message::getSize(const std::string& key) const { return content_->getSize(key); } void Message::getDoubleArray(const std::string& key, double* data, size_t len) const { return content_->getDoubleArray(key, data, len); } void Message::getFloatArray(const std::string& key, float* data, size_t len) const { return content_->getFloatArray(key, data, len); } eckit::Buffer Message::decode() const { return lookupDecoder().decode(*this); }; void Message::transform(const eckit::OrderedStringDict& dict) { content_->transform(dict); } eckit::DataHandle* Message::readHandle() const { return content_->readHandle(); } eckit::Offset Message::offset() const { return content_->offset(); } const void* Message::data() const { return content_->data(); } void Message::getMetadata(MetadataGatherer& gather, GetMetadataOptions options) const { return lookupDecoder().getMetadata(*this, gather, std::move(options)); } MessageDecoder& Message::lookupDecoder() const { if (decoder_ == nullptr) { decoder_ = &MessageDecoder::lookup(*this); } return *decoder_; } //---------------------------------------------------------------------------------------------------------------------- MetadataGatherer::~MetadataGatherer() {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::message eckit-2.0.7/src/eckit/message/Splitter.h0000664000175000017500000000503315161702250020274 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Jun 2020 #ifndef eckit_message_Splitter_h #define eckit_message_Splitter_h #include #include #include namespace eckit { class DataHandle; class PeekHandle; namespace message { class Message; //---------------------------------------------------------------------------------------------------------------------- class Splitter { public: // methods Splitter(eckit::PeekHandle&); virtual ~Splitter(); virtual Message next() = 0; protected: eckit::PeekHandle& handle_; private: // methods virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream& s, const Splitter& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- class SplitterBuilderBase { public: SplitterBuilderBase(); virtual ~SplitterBuilderBase(); virtual Splitter* make(eckit::PeekHandle&) const = 0; virtual bool match(eckit::PeekHandle&) const = 0; }; //---------------------------------------------------------------------------------------------------------------------- class SplitterFactory { public: static SplitterFactory& instance(); Splitter* lookup(eckit::PeekHandle&); void enregister(SplitterBuilderBase*); void deregister(const SplitterBuilderBase*); private: SplitterFactory() = default; ~SplitterFactory() = default; size_t index_ = 0; std::vector decoders_; // non-owning pointers std::mutex mutex_; }; //---------------------------------------------------------------------------------------------------------------------- template class SplitterBuilder : public SplitterBuilderBase { Splitter* make(eckit::PeekHandle& handle) const override { return new T(handle); } bool match(eckit::PeekHandle& handle) const override; // return M(handle); // } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace message } // namespace eckit #endif eckit-2.0.7/src/eckit/message/Reader.h0000664000175000017500000000276115161702250017675 0ustar alastairalastair/* * (C) Copyright 2017- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Aug 2017 #ifndef eckit_message_Reader_H #define eckit_message_Reader_H #include #include #include "eckit/io/PeekHandle.h" #include "eckit/message/Message.h" namespace eckit { class DataHandle; class PathName; class Offset; }; // namespace eckit namespace eckit::message { class Message; class Splitter; class Reader { public: Reader(eckit::DataHandle*, bool opened = false); Reader(eckit::DataHandle&, bool opened = false); Reader(const eckit::PathName&); Reader(const Reader&) = delete; Reader& operator=(const Reader&) = delete; Reader(Reader&&) = delete; Reader& operator=(Reader&&) = delete; ~Reader(); Message next(); eckit::Offset position(); private: std::unique_ptr splitter_; eckit::PeekHandle handle_; void init(); void print(std::ostream&) const; // Change to virtual if base class friend std::ostream& operator<<(std::ostream& s, const Reader& p) { p.print(s); return s; } }; } // namespace eckit::message #endif eckit-2.0.7/src/eckit/message/MessageContent.h0000664000175000017500000000460515161702250021411 0ustar alastairalastair/* * (C) Copyright 2017- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Jun 2020 #ifndef eckit_message_MessageContent_H #define eckit_message_MessageContent_H #include #include #include "eckit/memory/Counted.h" #include "eckit/types/Types.h" namespace eckit { class DataHandle; class Offset; class PathName; namespace mars { class MarsRequest; } namespace message { class MetadataGatherer; //---------------------------------------------------------------------------------------------------------------------- class Transformer { public: // virtual MessageContent* tranform(CodesContent*) const = 0; }; //---------------------------------------------------------------------------------------------------------------------- class MessageContent : public eckit::Counted { public: virtual operator bool() const; virtual void write(eckit::DataHandle&) const; virtual size_t length() const; virtual std::string getString(const std::string& key) const; virtual long getLong(const std::string& key) const; virtual double getDouble(const std::string& key) const; virtual void getDoubleArray(const std::string& key, std::vector&) const; virtual void getFloatArray(const std::string& key, std::vector&) const; virtual size_t getSize(const std::string& key) const; // Write double array at key to pre allocated array. virtual void getDoubleArray(const std::string& key, double* data, size_t len) const; virtual void getFloatArray(const std::string& key, float* data, size_t len) const; virtual eckit::DataHandle* readHandle() const; virtual eckit::Offset offset() const; virtual const void* data() const; virtual void transform(const Transformer&); virtual void transform(const eckit::OrderedStringDict&); private: virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream& s, const MessageContent& p) { p.print(s); return s; } }; } // namespace message } // namespace eckit #endif eckit-2.0.7/src/eckit/message/Reader.cc0000664000175000017500000000331015161702250020022 0ustar alastairalastair/* * (C) Copyright 2017- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/message/Reader.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/BufferedHandle.h" #include "eckit/message/Message.h" #include "eckit/message/Splitter.h" namespace { static size_t readerBufferSize() { static size_t readerBuffer = eckit::Resource("readerBuffer;$READER_BUFFER", 4 * 1024 * 1024); return readerBuffer; } } // namespace namespace eckit::message { Reader::Reader(eckit::DataHandle* h, bool opened) : // handle_(h), opened_(opened) { handle_(new BufferedHandle(h, readerBufferSize(), opened)) { init(); } Reader::Reader(eckit::DataHandle& h, bool opened) : // handle_(h), opened_(opened) { handle_(new BufferedHandle(h, readerBufferSize(), opened)) { init(); } Reader::Reader(const eckit::PathName& path) : handle_(new BufferedHandle(path.fileHandle(), readerBufferSize())) { init(); } void Reader::init() { handle_.openForRead(); splitter_.reset(SplitterFactory::instance().lookup(handle_)); } Reader::~Reader() { handle_.close(); } Message Reader::next() { return splitter_->next(); } void Reader::print(std::ostream& s) const { s << "Reader[" << handle_ << "," << *splitter_ << "]"; } eckit::Offset Reader::position() { return handle_.position(); } } // namespace eckit::message eckit-2.0.7/src/eckit/message/Decoder.h0000664000175000017500000000445515161702250020042 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Jun 2020 #ifndef eckit_message_Decoder_h #define eckit_message_Decoder_h #include #include #include "eckit/io/Buffer.h" #include "eckit/utils/EnumBitmask.h" namespace eckit::message { class Message; class MetadataGatherer; //---------------------------------------------------------------------------------------------------------------------- enum class ValueRepresentation : unsigned { Native = 0, String = 1, }; struct GetMetadataOptions { ValueRepresentation valueRepresentation{ValueRepresentation::String}; std::optional nameSpace{}; // Possible namespaces: // ls, statistics, parameter, time, geography, vertical, mars // (https://confluence.ecmwf.int/display/UDOC/What+are+namespaces+-+ecCodes+GRIB+FAQ) // Default: read gribToRequestNamespace from config, if not given use "mars". // To specify all namespaces, use "" }; //---------------------------------------------------------------------------------------------------------------------- class MessageDecoder { public: // methods MessageDecoder(); virtual ~MessageDecoder(); virtual void getMetadata(const Message& msg, MetadataGatherer& gatherer, const GetMetadataOptions& options = GetMetadataOptions{}) const = 0; virtual eckit::Buffer decode(const Message& msg) const = 0; static MessageDecoder& lookup(const Message&); private: // methods virtual bool match(const Message&) const = 0; virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream& s, const MessageDecoder& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::message #endif eckit-2.0.7/src/eckit/message/Splitter.cc0000664000175000017500000000500515161702250020431 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/message/Splitter.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/PeekHandle.h" #include "eckit/message/Message.h" #include #include namespace eckit::message { Splitter::Splitter(eckit::PeekHandle& handle) : handle_(handle) {} Splitter::~Splitter() {} //---------------------------------------------------------------------------------------------------------------------- SplitterFactory& SplitterFactory::instance() { static SplitterFactory theinstance; return theinstance; } void SplitterFactory::enregister(SplitterBuilderBase* b) { std::lock_guard lock(mutex_); decoders_.push_back(b); } void SplitterFactory::deregister(const SplitterBuilderBase* b) { std::lock_guard lock(mutex_); decoders_.erase(std::remove(decoders_.begin(), decoders_.end(), b), decoders_.end()); } SplitterBuilderBase::SplitterBuilderBase() { SplitterFactory::instance().enregister(this); } SplitterBuilderBase::~SplitterBuilderBase() { SplitterFactory::instance().deregister(this); } Splitter* SplitterFactory::lookup(eckit::PeekHandle& handle) { std::lock_guard lock(mutex_); size_t n = decoders_.size(); ASSERT(n > 0); for (size_t i = 0; i < n; ++i) { SplitterBuilderBase* builder = decoders_[(i + index_) % n]; if (builder->match(handle)) { index_ = i; // Start with this index for next message return builder->make(handle); } } std::ostringstream oss; oss << "Cannot find a metkit SplitterBuilder for " << handle << " "; for (size_t i = 0; i < handle.peeked(); ++i) { unsigned char c = handle.peek(i); oss << (isprint(c) ? char(c) : '.'); } oss << " - "; for (size_t i = 0; i < handle.peeked(); ++i) { unsigned char c = handle.peek(i); oss << std::setfill('0') << std::setw(2) << std::hex << int(c); } oss << std::endl; throw eckit::SeriousBug(oss.str()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::message eckit-2.0.7/src/eckit/contrib/0000775000175000017500000000000015161702250016330 5ustar alastairalastaireckit-2.0.7/src/eckit/contrib/xxhash/0000775000175000017500000000000015161702250017633 5ustar alastairalastaireckit-2.0.7/src/eckit/contrib/xxhash/eckit-manifest.txt0000664000175000017500000000036015161702250023276 0ustar alastairalastairdate: Downloaded on Feb 9, 2021 git url: https://github.com/Cyan4973/xxHash git commit sha1: f2c52f1236a50d754b07f584ce4592de1df8c0f7 git branch: dev version: further developments after 0.8.0 (July 27, 2020) eckit-2.0.7/src/eckit/contrib/xxhash/xxhash.h0000664000175000017500000062122615161702250021320 0ustar alastairalastair/* * xxHash - Extremely Fast Hash algorithm * Header File * Copyright (C) 2012-2020 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * You can contact the author at: * - xxHash homepage: https://www.xxhash.com * - xxHash source repository: https://github.com/Cyan4973/xxHash */ /*! * @mainpage xxHash * * @file xxhash.h * xxHash prototypes and implementation */ /* TODO: update */ /* Notice extracted from xxHash homepage: xxHash is an extremely fast hash algorithm, running at RAM speed limits. It also successfully passes all tests from the SMHasher suite. Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) Name Speed Q.Score Author xxHash 5.4 GB/s 10 CrapWow 3.2 GB/s 2 Andrew MurmurHash 3a 2.7 GB/s 10 Austin Appleby SpookyHash 2.0 GB/s 10 Bob Jenkins SBox 1.4 GB/s 9 Bret Mulvey Lookup3 1.2 GB/s 9 Bob Jenkins SuperFastHash 1.2 GB/s 1 Paul Hsieh CityHash64 1.05 GB/s 10 Pike & Alakuijala FNV 0.55 GB/s 5 Fowler, Noll, Vo CRC32 0.43 GB/s 9 MD5-32 0.33 GB/s 10 Ronald L. Rivest SHA1-32 0.28 GB/s 10 Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. Note: SMHasher's CRC32 implementation is not the fastest one. Other speed-oriented implementations can be faster, especially in combination with PCLMUL instruction: https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 A 64-bit version, named XXH64, is available since r35. It offers much better speed, but for 64-bit applications only. Name Speed on 64 bits Speed on 32 bits XXH64 13.8 GB/s 1.9 GB/s XXH32 6.8 GB/s 6.0 GB/s */ #if defined(__cplusplus) extern "C" { #endif /* **************************** * INLINE mode ******************************/ /*! * XXH_INLINE_ALL (and XXH_PRIVATE_API) * Use these build macros to inline xxhash into the target unit. * Inlining improves performance on small inputs, especially when the length is * expressed as a compile-time constant: * * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html * * It also keeps xxHash symbols private to the unit, so they are not exported. * * Usage: * #define XXH_INLINE_ALL * #include "xxhash.h" * * Do not compile and link xxhash.o as a separate object, as it is not useful. */ #if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) && !defined(XXH_INLINE_ALL_31684351384) /* this section should be traversed only once */ #define XXH_INLINE_ALL_31684351384 /* give access to the advanced API, required to compile implementations */ #undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ #define XXH_STATIC_LINKING_ONLY /* make all functions private */ #undef XXH_PUBLIC_API #if defined(__GNUC__) #define XXH_PUBLIC_API static __inline __attribute__((unused)) #elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) #define XXH_PUBLIC_API static inline #elif defined(_MSC_VER) #define XXH_PUBLIC_API static __inline #else /* note: this version may generate warnings for unused static functions */ #define XXH_PUBLIC_API static #endif /* * This part deals with the special case where a unit wants to inline xxHash, * but "xxhash.h" has previously been included without XXH_INLINE_ALL, such * as part of some previously included *.h header file. * Without further action, the new include would just be ignored, * and functions would effectively _not_ be inlined (silent failure). * The following macros solve this situation by prefixing all inlined names, * avoiding naming collision with previous inclusions. */ #ifdef XXH_NAMESPACE #error "XXH_INLINE_ALL with XXH_NAMESPACE is not supported" /* * Note: Alternative: #undef all symbols (it's a pretty large list). * Without #error: it compiles, but functions are actually not inlined. */ #endif #define XXH_NAMESPACE XXH_INLINE_ /* * Some identifiers (enums, type names) are not symbols, but they must * still be renamed to avoid redeclaration. * Alternative solution: do not redeclare them. * However, this requires some #ifdefs, and is a more dispersed action. * Meanwhile, renaming can be achieved in a single block */ #define XXH_IPREF(Id) XXH_INLINE_##Id #define XXH_OK XXH_IPREF(XXH_OK) #define XXH_ERROR XXH_IPREF(XXH_ERROR) #define XXH_errorcode XXH_IPREF(XXH_errorcode) #define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) #define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) #define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) #define XXH32_state_s XXH_IPREF(XXH32_state_s) #define XXH32_state_t XXH_IPREF(XXH32_state_t) #define XXH64_state_s XXH_IPREF(XXH64_state_s) #define XXH64_state_t XXH_IPREF(XXH64_state_t) #define XXH3_state_s XXH_IPREF(XXH3_state_s) #define XXH3_state_t XXH_IPREF(XXH3_state_t) #define XXH128_hash_t XXH_IPREF(XXH128_hash_t) /* Ensure the header is parsed again, even if it was previously included */ #undef XXHASH_H_5627135585666179 #undef XXHASH_H_STATIC_13879238742 #endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ /* **************************************************************** * Stable API *****************************************************************/ #ifndef XXHASH_H_5627135585666179 #define XXHASH_H_5627135585666179 1 /*! * @defgroup public Public API * Contains details on the public xxHash functions. * @{ */ /* specific declaration modes for Windows */ #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) #if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) #ifdef XXH_EXPORT #define XXH_PUBLIC_API __declspec(dllexport) #elif XXH_IMPORT #define XXH_PUBLIC_API __declspec(dllimport) #endif #else #define XXH_PUBLIC_API /* do nothing */ #endif #endif #ifdef XXH_DOXYGEN /*! * @brief Emulate a namespace by transparently prefixing all symbols. * * If you want to include _and expose_ xxHash functions from within your own * library, but also want to avoid symbol collisions with other libraries which * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix * any public symbol from xxhash library with the value of XXH_NAMESPACE * (therefore, avoid empty or numeric values). * * Note that no change is required within the calling program as long as it * includes `xxhash.h`: Regular symbol names will be automatically translated * by this header. */ #define XXH_NAMESPACE /* YOUR NAME HERE */ #undef XXH_NAMESPACE #endif #ifdef XXH_NAMESPACE #define XXH_CAT(A, B) A##B #define XXH_NAME2(A, B) XXH_CAT(A, B) #define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) /* XXH32 */ #define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) #define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) #define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) #define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) #define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) #define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) #define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) #define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) #define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) /* XXH64 */ #define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) #define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) #define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) #define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) #define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) #define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) #define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) #define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) #define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) /* XXH3_64bits */ #define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) #define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) #define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) #define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) #define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) #define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) #define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) #define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) #define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) #define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) #define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) #define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) /* XXH3_128bits */ #define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) #define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) #define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) #define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) #define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) #define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) #define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) #define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) #define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) #define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) #define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) #define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) #define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) #endif /* ************************************* * Version ***************************************/ #define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MINOR 8 #define XXH_VERSION_RELEASE 0 #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR * 100 * 100 + XXH_VERSION_MINOR * 100 + XXH_VERSION_RELEASE) /*! * @brief Obtains the xxHash version. * * This is only useful when xxHash is compiled as a shared library, as it is * independent of the version defined in the header. * * @return `XXH_VERSION_NUMBER` as of when the function was compiled. */ XXH_PUBLIC_API unsigned XXH_versionNumber(void); /* **************************** * Definitions ******************************/ #include /* size_t */ typedef enum { XXH_OK = 0, XXH_ERROR } XXH_errorcode; /*-********************************************************************** * 32-bit hash ************************************************************************/ #if defined(XXH_DOXYGEN) /* Don't show include */ /*! * @brief An unsigned 32-bit integer. * * Not necessarily defined to `uint32_t` but functionally equivalent. */ typedef uint32_t XXH32_hash_t; #elif !defined(__VMS) && \ (defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) #include typedef uint32_t XXH32_hash_t; #else #include #if UINT_MAX == 0xFFFFFFFFUL typedef unsigned int XXH32_hash_t; #else #if ULONG_MAX == 0xFFFFFFFFUL typedef unsigned long XXH32_hash_t; #else #error "unsupported platform: need a 32-bit type" #endif #endif #endif /*! * @} * * @defgroup xxh32_family XXH32 family * @ingroup public * Contains functions used in the classic 32-bit xxHash algorithm. * * @note * XXH32 is considered rather weak by today's standards. * The @ref xxh3_family provides competitive speed for both 32-bit and 64-bit * systems, and offers true 64/128 bit hash results. It provides a superior * level of dispersion, and greatly reduces the risks of collisions. * * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families * @see @ref xxh32_impl for implementation details * @{ */ /*! * @brief Calculates the 32-bit hash of @p input using xxHash32. * * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s * * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * @param seed The 32-bit seed to alter the hash's output predictably. * * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return The calculated 32-bit hash value. * * @see * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): * Direct equivalents for the other variants of xxHash. * @see * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. */ XXH_PUBLIC_API XXH32_hash_t XXH32(const void* input, size_t length, XXH32_hash_t seed); /*! * Streaming functions generate the xxHash value from an incremental input. * This method is slower than single-call functions, due to state management. * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. * * An XXH state must first be allocated using `XXH*_createState()`. * * Start a new hash by initializing the state with a seed using `XXH*_reset()`. * * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. * * The function returns an error code, with 0 meaning OK, and any other value * meaning there is an error. * * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. * This function returns the nn-bits hash as an int or long long. * * It's still possible to continue inserting input into the hash state after a * digest, and generate new hash values later on by invoking `XXH*_digest()`. * * When done, release the state using `XXH*_freeState()`. * * Example code for incrementally hashing a file: * @code{.c} * #include * #include * #define BUFFER_SIZE 256 * * // Note: XXH64 and XXH3 use the same interface. * XXH32_hash_t * hashFile(FILE* stream) * { * XXH32_state_t* state; * unsigned char buf[BUFFER_SIZE]; * size_t amt; * XXH32_hash_t hash; * * state = XXH32_createState(); // Create a state * assert(state != NULL); // Error check here * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { * XXH32_update(state, buf, amt); // Hash the file in chunks * } * hash = XXH32_digest(state); // Finalize the hash * XXH32_freeState(state); // Clean up * return hash; * } * @endcode */ /*! * @typedef struct XXH32_state_s XXH32_state_t * @brief The opaque state struct for the XXH32 streaming API. * * @see XXH32_state_s for details. */ typedef struct XXH32_state_s XXH32_state_t; /*! * @brief Allocates an @ref XXH32_state_t. * * Must be freed with XXH32_freeState(). * @return An allocated XXH32_state_t on success, `NULL` on failure. */ XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); /*! * @brief Frees an @ref XXH32_state_t. * * Must be allocated with XXH32_createState(). * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). * @return XXH_OK. */ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); /*! * @brief Copies one @ref XXH32_state_t to another. * * @param dst_state The state to copy to. * @param src_state The state to copy from. * @pre * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); /*! * @brief Resets an @ref XXH32_state_t to begin a new hash. * * This function resets and seeds a state. Call it before @ref XXH32_update(). * * @param statePtr The state struct to reset. * @param seed The 32-bit seed to alter the hash result predictably. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. */ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed); /*! * @brief Consumes a block of @p input to an @ref XXH32_state_t. * * Call this to incrementally consume blocks of data. * * @param statePtr The state struct to update. * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * * @pre * @p statePtr must not be `NULL`. * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. */ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t* statePtr, const void* input, size_t length); /*! * @brief Returns the calculated hash value from an @ref XXH32_state_t. * * @note * Calling XXH32_digest() will not affect @p statePtr, so you can update, * digest, and update again. * * @param statePtr The state struct to calculate the hash from. * * @pre * @p statePtr must not be `NULL`. * * @return The calculated xxHash32 value from that state. */ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* statePtr); /******* Canonical representation *******/ /* * The default return values from XXH functions are unsigned 32 and 64 bit * integers. * This the simplest and fastest format for further post-processing. * * However, this leaves open the question of what is the order on the byte level, * since little and big endian conventions will store the same number differently. * * The canonical representation settles this issue by mandating big-endian * convention, the same convention as human-readable numbers (large digits first). * * When writing hash values to storage, sending them over a network, or printing * them, it's highly recommended to use the canonical representation to ensure * portability across a wider range of systems, present and future. * * The following functions allow transformation of hash values to and from * canonical format. */ /*! * @brief Canonical (big endian) representation of @ref XXH32_hash_t. */ typedef struct { unsigned char digest[4]; /*!< Hash bytes, big endian */ } XXH32_canonical_t; /*! * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. * * @param dst The @ref XXH32_canonical_t pointer to be stored to. * @param hash The @ref XXH32_hash_t to be converted. * * @pre * @p dst must not be `NULL`. */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); /*! * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. * * @param src The @ref XXH32_canonical_t to convert. * * @pre * @p src must not be `NULL`. * * @return The converted hash. */ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); /*! * @} * @ingroup public * @{ */ #ifndef XXH_NO_LONG_LONG /*-********************************************************************** * 64-bit hash ************************************************************************/ #if defined(XXH_DOXYGEN) /* don't include */ /*! * @brief An unsigned 64-bit integer. * * Not necessarily defined to `uint64_t` but functionally equivalent. */ typedef uint64_t XXH64_hash_t; #elif !defined(__VMS) && \ (defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) #include typedef uint64_t XXH64_hash_t; #else #include #if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL /* LP64 ABI says uint64_t is unsigned long */ typedef unsigned long XXH64_hash_t; #else /* the following type must have a width of 64-bit */ typedef unsigned long long XXH64_hash_t; #endif #endif /*! * @} * * @defgroup xxh64_family XXH64 family * @ingroup public * @{ * Contains functions used in the classic 64-bit xxHash algorithm. * * @note * XXH3 provides competitive speed for both 32-bit and 64-bit systems, * and offers true 64/128 bit hash results. It provides a superior level of * dispersion, and greatly reduces the risks of collisions. */ /*! * @brief Calculates the 64-bit hash of @p input using xxHash64. * * This function usually runs faster on 64-bit systems, but slower on 32-bit * systems (see benchmark). * * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * @param seed The 64-bit seed to alter the hash's output predictably. * * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return The calculated 64-bit hash. * * @see * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): * Direct equivalents for the other variants of xxHash. * @see * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. */ XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); /******* Streaming *******/ /*! * @brief The opaque state struct for the XXH64 streaming API. * * @see XXH64_state_s for details. */ typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed); XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t* statePtr, const void* input, size_t length); XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* statePtr); /******* Canonical representation *******/ typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); /*! * @} * ************************************************************************ * @defgroup xxh3_family XXH3 family * @ingroup public * @{ * * XXH3 is a more recent hash algorithm featuring: * - Improved speed for both small and large inputs * - True 64-bit and 128-bit outputs * - SIMD acceleration * - Improved 32-bit viability * * Speed analysis methodology is explained here: * * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html * * Compared to XXH64, expect XXH3 to run approximately * ~2x faster on large inputs and >3x faster on small ones, * exact differences vary depending on platform. * * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, * but does not require it. * Any 32-bit and 64-bit targets that can run XXH32 smoothly * can run XXH3 at competitive speeds, even without vector support. * Further details are explained in the implementation. * * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. * * XXH3 implementation is portable: * it has a generic C90 formulation that can be compiled on any platform, * all implementations generage exactly the same hash value on all platforms. * Starting from v0.8.0, it's also labelled "stable", meaning that * any future version will also generate the same hash value. * * XXH3 offers 2 variants, _64bits and _128bits. * * When only 64 bits are needed, prefer invoking the _64bits variant, as it * reduces the amount of mixing, resulting in faster speed on small inputs. * It's also generally simpler to manipulate a scalar return type than a struct. * * The API supports one-shot hashing, streaming mode, and custom secrets. */ /*-********************************************************************** * XXH3 64-bit variant ************************************************************************/ /* XXH3_64bits(): * default 64-bit variant, using default secret and default seed of 0. * It's the fastest variant. */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); /* * XXH3_64bits_withSeed(): * This variant generates a custom secret on the fly * based on default secret altered using the `seed` value. * While this operation is decently fast, note that it's not completely free. * Note: seed==0 produces the same results as XXH3_64bits(). */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); /*! * The bare minimum size for a custom secret. * * @see * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). */ #define XXH3_SECRET_SIZE_MIN 136 /* * XXH3_64bits_withSecret(): * It's possible to provide any blob of bytes as a "secret" to generate the hash. * This makes it more difficult for an external actor to prepare an intentional collision. * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). * However, the quality of produced hash values depends on secret's entropy. * Technically, the secret must look like a bunch of random bytes. * Avoid "trivial" or structured data such as repeated sequences or a text document. * Whenever unsure about the "randomness" of the blob of bytes, * consider relabelling it as a "custom seed" instead, * and employ "XXH3_generateSecret()" (see below) * to generate a high entropy secret derived from the custom seed. */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); /******* Streaming *******/ /* * Streaming requires state maintenance. * This operation costs memory and CPU. * As a consequence, streaming is slower than one-shot hashing. * For better performance, prefer one-shot functions whenever applicable. */ /*! * @brief The state struct for the XXH3 streaming API. * * @see XXH3_state_s for details. */ typedef struct XXH3_state_s XXH3_state_t; XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); /* * XXH3_64bits_reset(): * Initialize with default parameters. * digest will be equivalent to `XXH3_64bits()`. */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); /* * XXH3_64bits_reset_withSeed(): * Generate a custom secret from `seed`, and store it into `statePtr`. * digest will be equivalent to `XXH3_64bits_withSeed()`. */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); /* * XXH3_64bits_reset_withSecret(): * `secret` is referenced, it _must outlive_ the hash streaming session. * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, * and the quality of produced hash values depends on secret's entropy * (secret's content should look like a bunch of random bytes). * When in doubt about the randomness of a candidate `secret`, * consider employing `XXH3_generateSecret()` instead (see below). */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update(XXH3_state_t* statePtr, const void* input, size_t length); XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t* statePtr); /* note : canonical representation of XXH3 is the same as XXH64 * since they both produce XXH64_hash_t values */ /*-********************************************************************** * XXH3 128-bit variant ************************************************************************/ /*! * @brief The return value from 128-bit hashes. * * Stored in little endian order, although the fields themselves are in native * endianness. */ typedef struct { XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ XXH64_hash_t high64; /*!< `value >> 64` */ } XXH128_hash_t; XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); /******* Streaming *******/ /* * Streaming requires state maintenance. * This operation costs memory and CPU. * As a consequence, streaming is slower than one-shot hashing. * For better performance, prefer one-shot functions whenever applicable. * * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). * Use already declared XXH3_createState() and XXH3_freeState(). * * All reset and streaming functions have same meaning as their 64-bit counterpart. */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t* statePtr, const void* input, size_t length); XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t* statePtr); /* Following helper functions make it possible to compare XXH128_hast_t values. * Since XXH128_hash_t is a structure, this capability is not offered by the language. * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ /*! * XXH128_isEqual(): * Return: 1 if `h1` and `h2` are equal, 0 if they are not. */ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); /*! * XXH128_cmp(): * * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. * * return: >0 if *h128_1 > *h128_2 * =0 if *h128_1 == *h128_2 * <0 if *h128_1 < *h128_2 */ XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); /******* Canonical representation *******/ typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); #endif /* XXH_NO_LONG_LONG */ /*! * @} */ #endif /* XXHASH_H_5627135585666179 */ #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) #define XXHASH_H_STATIC_13879238742 /* **************************************************************************** * This section contains declarations which are not guaranteed to remain stable. * They may change in future versions, becoming incompatible with a different * version of the library. * These declarations should only be used with static linking. * Never use them in association with dynamic linking! ***************************************************************************** */ /* * These definitions are only present to allow static allocation * of XXH states, on stack or in a struct, for example. * Never **ever** access their members directly. */ /*! * @internal * @brief Structure for XXH32 streaming API. * * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is * an opaque type. This allows fields to safely be changed. * * Typedef'd to @ref XXH32_state_t. * Do not access the members of this struct directly. * @see XXH64_state_s, XXH3_state_s */ struct XXH32_state_s { XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ XXH32_hash_t v1; /*!< First accumulator lane */ XXH32_hash_t v2; /*!< Second accumulator lane */ XXH32_hash_t v3; /*!< Third accumulator lane */ XXH32_hash_t v4; /*!< Fourth accumulator lane */ XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may be removed. */ }; /* typedef'd to XXH32_state_t */ #ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ /*! * @internal * @brief Structure for XXH64 streaming API. * * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is * an opaque type. This allows fields to safely be changed. * * Typedef'd to @ref XXH64_state_t. * Do not access the members of this struct directly. * @see XXH32_state_s, XXH3_state_s */ struct XXH64_state_s { XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ XXH64_hash_t v1; /*!< First accumulator lane */ XXH64_hash_t v2; /*!< Second accumulator lane */ XXH64_hash_t v3; /*!< Third accumulator lane */ XXH64_hash_t v4; /*!< Fourth accumulator lane */ XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it may be removed. */ }; /* typedef'd to XXH64_state_t */ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11+ */ #include #define XXH_ALIGN(n) alignas(n) #elif defined(__GNUC__) #define XXH_ALIGN(n) __attribute__((aligned(n))) #elif defined(_MSC_VER) #define XXH_ALIGN(n) __declspec(align(n)) #else #define XXH_ALIGN(n) /* disabled */ #endif /* Old GCC versions only accept the attribute after the type in structures. */ #if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ && defined(__GNUC__) #define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) #else #define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type #endif /*! * @brief The size of the internal XXH3 buffer. * * This is the optimal update size for incremental hashing. * * @see XXH3_64b_update(), XXH3_128b_update(). */ #define XXH3_INTERNALBUFFER_SIZE 256 /*! * @brief Default size of the secret buffer (and @ref XXH3_kSecret). * * This is the size used in @ref XXH3_kSecret and the seeded functions. * * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. */ #define XXH3_SECRET_DEFAULT_SIZE 192 /*! * @internal * @brief Structure for XXH3 streaming API. * * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is * an opaque type. This allows fields to safely be changed. * * @note **This structure has a strict alignment requirement of 64 bytes.** Do * not allocate this with `malloc()` or `new`, it will not be sufficiently * aligned. Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack * allocation. * * Typedef'd to @ref XXH3_state_t. * Do not access the members of this struct directly. * * @see XXH3_INITSTATE() for stack initialization. * @see XXH3_createState(), XXH3_freeState(). * @see XXH32_state_s, XXH64_state_s */ struct XXH3_state_s { XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */ XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); /*!< Used to store a custom secret generated from a seed. */ XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); /*!< The internal buffer. @see XXH32_state_s::mem32 */ XXH32_hash_t bufferedSize; /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ XXH32_hash_t reserved32; /*!< Reserved field. Needed for padding on 64-bit. */ size_t nbStripesSoFar; /*!< Number or stripes processed. */ XXH64_hash_t totalLen; /*!< Total length hashed. 64-bit even on 32-bit targets. */ size_t nbStripesPerBlock; /*!< Number of stripes per block. */ size_t secretLimit; /*!< Size of @ref customSecret or @ref extSecret */ XXH64_hash_t seed; /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ XXH64_hash_t reserved64; /*!< Reserved field. */ const unsigned char* extSecret; /*!< Reference to an external secret for the _withSecret variants, NULL * for other variants. */ /* note: there may be some padding at the end due to alignment on 64 bytes */ }; /* typedef'd to XXH3_state_t */ #undef XXH_ALIGN_MEMBER /*! * @brief Initializes a stack-allocated `XXH3_state_s`. * * When the @ref XXH3_state_t structure is merely emplaced on stack, * it should be initialized with XXH3_INITSTATE() or a memset() * in case its first reset uses XXH3_NNbits_reset_withSeed(). * This init can be omitted if the first reset uses default or _withSecret mode. * This operation isn't necessary when the state is created with XXH3_createState(). * Note that this doesn't prepare the state for a streaming operation, * it's still necessary to use XXH3_NNbits_reset*() afterwards. */ #define XXH3_INITSTATE(XXH3_state_ptr) \ { (XXH3_state_ptr)->seed = 0; } /* === Experimental API === */ /* Symbols defined below must be considered tied to a specific library version. */ /* * XXH3_generateSecret(): * * Derive a high-entropy secret from any user-defined content, named customSeed. * The generated secret can be used in combination with `*_withSecret()` functions. * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed, * as it becomes much more difficult for an external actor to guess how to impact the calculation logic. * * The function accepts as input a custom seed of any length and any content, * and derives from it a high-entropy secret of length XXH3_SECRET_DEFAULT_SIZE * into an already allocated buffer secretBuffer. * The generated secret is _always_ XXH_SECRET_DEFAULT_SIZE bytes long. * * The generated secret can then be used with any `*_withSecret()` variant. * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` * are part of this list. They all accept a `secret` parameter * which must be very long for implementation reasons (>= XXH3_SECRET_SIZE_MIN) * _and_ feature very high entropy (consist of random-looking bytes). * These conditions can be a high bar to meet, so * this function can be used to generate a secret of proper quality. * * customSeed can be anything. It can have any size, even small ones, * and its content can be anything, even stupidly "low entropy" source such as a bunch of zeroes. * The resulting `secret` will nonetheless provide all expected qualities. * * Supplying NULL as the customSeed copies the default secret into `secretBuffer`. * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior. */ XXH_PUBLIC_API void XXH3_generateSecret(void* secretBuffer, const void* customSeed, size_t customSeedSize); /* simple short-cut to pre-selected XXH3_128bits variant */ XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); #endif /* XXH_NO_LONG_LONG */ #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) #define XXH_IMPLEMENTATION #endif #endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ /* ======================================================================== */ /* ======================================================================== */ /* ======================================================================== */ /*-********************************************************************** * xxHash implementation *-********************************************************************** * xxHash's implementation used to be hosted inside xxhash.c. * * However, inlining requires implementation to be visible to the compiler, * hence be included alongside the header. * Previously, implementation was hosted inside xxhash.c, * which was then #included when inlining was activated. * This construction created issues with a few build and install systems, * as it required xxhash.c to be stored in /include directory. * * xxHash implementation is now directly integrated within xxhash.h. * As a consequence, xxhash.c is no longer needed in /include. * * xxhash.c is still available and is still useful. * In a "normal" setup, when xxhash is not inlined, * xxhash.h only exposes the prototypes and public symbols, * while xxhash.c can be built into an object file xxhash.o * which can then be linked into the final binary. ************************************************************************/ #if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) || defined(XXH_IMPLEMENTATION)) && \ !defined(XXH_IMPLEM_13a8737387) #define XXH_IMPLEM_13a8737387 /* ************************************* * Tuning parameters ***************************************/ /*! * @defgroup tuning Tuning parameters * @{ * * Various macros to control xxHash's behavior. */ #ifdef XXH_DOXYGEN /*! * @brief Define this to disable 64-bit code. * * Useful if only using the @ref xxh32_family and you have a strict C90 compiler. */ #define XXH_NO_LONG_LONG #undef XXH_NO_LONG_LONG /* don't actually */ /*! * @brief Controls how unaligned memory is accessed. * * By default, access to unaligned memory is controlled by `memcpy()`, which is * safe and portable. * * Unfortunately, on some target/compiler combinations, the generated assembly * is sub-optimal. * * The below switch allow selection of a different access method * in the search for improved performance. * * @par Possible options: * * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` * @par * Use `memcpy()`. Safe and portable. Note that most modern compilers will * eliminate the function call and treat it as an unaligned access. * * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` * @par * Depends on compiler extensions and is therefore not portable. * This method is safe if your compiler supports it, and *generally* as * fast or faster than `memcpy`. * * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast * @par * Casts directly and dereferences. This method doesn't depend on the * compiler, but it violates the C standard as it directly dereferences an * unaligned pointer. It can generate buggy code on targets which do not * support unaligned memory accesses, but in some circumstances, it's the * only known way to get the most performance (example: GCC + ARMv6). * * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift * @par * Also portable. This can generate the best code on old compilers which don't * inline small `memcpy()` calls, and it might also be faster on big-endian * systems which lack a native byteswap instruction. However, some compilers * will emit literal byteshifts even if the target supports unaligned access. * * . * * @warning * Methods 1 and 2 rely on implementation-defined behavior. Use these with * care, as what works on one compiler/platform/optimization level may cause * another to read garbage data or even crash. * * See https://stackoverflow.com/a/32095106/646947 for details. * * Prefer these methods in priority order (0 > 3 > 1 > 2) */ #define XXH_FORCE_MEMORY_ACCESS 0 /*! * @def XXH_ACCEPT_NULL_INPUT_POINTER * @brief Whether to add explicit `NULL` checks. * * If the input pointer is `NULL` and the length is non-zero, xxHash's default * behavior is to dereference it, triggering a segfault. * * When this macro is enabled, xxHash actively checks the input for a null pointer. * If it is, the result for null input pointers is the same as a zero-length input. */ #define XXH_ACCEPT_NULL_INPUT_POINTER 0 /*! * @def XXH_FORCE_ALIGN_CHECK * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() * and XXH64() only). * * This is an important performance trick for architectures without decent * unaligned memory access performance. * * It checks for input alignment, and when conditions are met, uses a "fast * path" employing direct 32-bit/64-bit reads, resulting in _dramatically * faster_ read speed. * * The check costs one initial branch per hash, which is generally negligible, * but not zero. * * Moreover, it's not useful to generate an additional code path if memory * access uses the same instruction for both aligned and unaligned * addresses (e.g. x86 and aarch64). * * In these cases, the alignment check can be removed by setting this macro to 0. * Then the code will always use unaligned memory access. * Align check is automatically disabled on x86, x64 & arm64, * which are platforms known to offer good unaligned memory accesses performance. * * This option does not affect XXH3 (only XXH32 and XXH64). */ #define XXH_FORCE_ALIGN_CHECK 0 /*! * @def XXH_NO_INLINE_HINTS * @brief When non-zero, sets all functions to `static`. * * By default, xxHash tries to force the compiler to inline almost all internal * functions. * * This can usually improve performance due to reduced jumping and improved * constant folding, but significantly increases the size of the binary which * might not be favorable. * * Additionally, sometimes the forced inlining can be detrimental to performance, * depending on the architecture. * * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the * compiler full control on whether to inline or not. * * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using * -fno-inline with GCC or Clang, this will automatically be defined. */ #define XXH_NO_INLINE_HINTS 0 /*! * @def XXH_REROLL * @brief Whether to reroll `XXH32_finalize` and `XXH64_finalize`. * * For performance, `XXH32_finalize` and `XXH64_finalize` use an unrolled loop * in the form of a switch statement. * * This is not always desirable, as it generates larger code, and depending on * the architecture, may even be slower * * This is automatically defined with `-Os`/`-Oz` on GCC and Clang. */ #define XXH_REROLL 0 /*! * @internal * @brief Redefines old internal names. * * For compatibility with code that uses xxHash's internals before the names * were changed to improve namespacing. There is no other reason to use this. */ #define XXH_OLD_NAMES #undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ #endif /* XXH_DOXYGEN */ /*! * @} */ #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ #if !defined(__clang__) && defined(__GNUC__) && defined(__ARM_FEATURE_UNALIGNED) && defined(__ARM_ARCH) && \ (__ARM_ARCH == 6) #define XXH_FORCE_MEMORY_ACCESS 2 #elif !defined(__clang__) && ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ (defined(__GNUC__) && (defined(__ARM_ARCH) && __ARM_ARCH >= 7))) #define XXH_FORCE_MEMORY_ACCESS 1 #endif #endif #ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ #define XXH_ACCEPT_NULL_INPUT_POINTER 0 #endif #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ #if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(_M_IX86) || defined(_M_X64) || \ defined(_M_ARM64) /* visual */ #define XXH_FORCE_ALIGN_CHECK 0 #else #define XXH_FORCE_ALIGN_CHECK 1 #endif #endif #ifndef XXH_NO_INLINE_HINTS #if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ || defined(__NO_INLINE__) /* -O0, -fno-inline */ #define XXH_NO_INLINE_HINTS 1 #else #define XXH_NO_INLINE_HINTS 0 #endif #endif #ifndef XXH_REROLL #if defined(__OPTIMIZE_SIZE__) #define XXH_REROLL 1 #else #define XXH_REROLL 0 #endif #endif /*! * @defgroup impl Implementation * @{ */ /* ************************************* * Includes & Memory related functions ***************************************/ /* * Modify the local functions below should you wish to use * different memory routines for malloc() and free() */ #include /*! * @internal * @brief Modify this function to use a different routine than malloc(). */ static void* XXH_malloc(size_t s) { return malloc(s); } /*! * @internal * @brief Modify this function to use a different routine than free(). */ static void XXH_free(void* p) { free(p); } #include /*! * @internal * @brief Modify this function to use a different routine than memcpy(). */ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest, src, size); } #include /* ULLONG_MAX */ /* ************************************* * Compiler Specific Options ***************************************/ #ifdef _MSC_VER /* Visual Studio warning fix */ #pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif #if XXH_NO_INLINE_HINTS /* disable inlining hints */ #if defined(__GNUC__) #define XXH_FORCE_INLINE static __attribute__((unused)) #else #define XXH_FORCE_INLINE static #endif #define XXH_NO_INLINE static /* enable inlining hints */ #elif defined(_MSC_VER) /* Visual Studio */ #define XXH_FORCE_INLINE static __forceinline #define XXH_NO_INLINE static __declspec(noinline) #elif defined(__GNUC__) #define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) #define XXH_NO_INLINE static __attribute__((noinline)) #elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ #define XXH_FORCE_INLINE static inline #define XXH_NO_INLINE static #else #define XXH_FORCE_INLINE static #define XXH_NO_INLINE static #endif /* ************************************* * Debug ***************************************/ /*! * @ingroup tuning * @def XXH_DEBUGLEVEL * @brief Sets the debugging level. * * XXH_DEBUGLEVEL is expected to be defined externally, typically via the * compiler's command line options. The value must be a number. */ #ifndef XXH_DEBUGLEVEL #ifdef DEBUGLEVEL /* backwards compat */ #define XXH_DEBUGLEVEL DEBUGLEVEL #else #define XXH_DEBUGLEVEL 0 #endif #endif #if (XXH_DEBUGLEVEL >= 1) #include /* note: can still be disabled with NDEBUG */ #define XXH_ASSERT(c) assert(c) #else #define XXH_ASSERT(c) ((void)0) #endif /* note: use after variable declarations */ #define XXH_STATIC_ASSERT(c) \ do { \ enum { \ XXH_sa = 1 / (int)(!!(c)) \ }; \ } while (0) /* ************************************* * Basic Types ***************************************/ #if !defined(__VMS) && (defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 \ */)) #include typedef uint8_t xxh_u8; #else typedef unsigned char xxh_u8; #endif typedef XXH32_hash_t xxh_u32; #ifdef XXH_OLD_NAMES #define BYTE xxh_u8 #define U8 xxh_u8 #define U32 xxh_u32 #endif /* *** Memory access *** */ /*! * @internal * @fn xxh_u32 XXH_read32(const void* ptr) * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. * * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * * @param ptr The pointer to read from. * @return The 32-bit native endian integer from the bytes at @p ptr. */ /*! * @internal * @fn xxh_u32 XXH_readLE32(const void* ptr) * @brief Reads an unaligned 32-bit little endian integer from @p ptr. * * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * * @param ptr The pointer to read from. * @return The 32-bit little endian integer from the bytes at @p ptr. */ /*! * @internal * @fn xxh_u32 XXH_readBE32(const void* ptr) * @brief Reads an unaligned 32-bit big endian integer from @p ptr. * * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * * @param ptr The pointer to read from. * @return The 32-bit big endian integer from the bytes at @p ptr. */ /*! * @internal * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. * * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is * always @ref XXH_alignment::XXH_unaligned. * * @param ptr The pointer to read from. * @param align Whether @p ptr is aligned. * @pre * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte * aligned. * @return The 32-bit little endian integer from the bytes at @p ptr. */ #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) /* * Manual byteshift. Best for old compilers which don't inline memcpy. * We actually directly use XXH_readLE32 and XXH_readBE32. */ #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 2)) /* * Force direct memory access. Only works on CPU which support unaligned memory * access in hardware. */ static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*)memPtr; } #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 1)) /* * __pack instructions are safer but compiler specific, hence potentially * problematic for some compilers. * * Currently only defined for GCC and ICC. */ #ifdef XXH_OLD_NAMES typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; #endif static xxh_u32 XXH_read32(const void* ptr) { typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign; return ((const xxh_unalign*)ptr)->u32; } #else /* * Portable and safe solution. Generally efficient. * see: https://stackoverflow.com/a/32095106/646947 */ static xxh_u32 XXH_read32(const void* memPtr) { xxh_u32 val; memcpy(&val, memPtr, sizeof(val)); return val; } #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ /* *** Endianness *** */ typedef enum { XXH_bigEndian = 0, XXH_littleEndian = 1 } XXH_endianess; /*! * @ingroup tuning * @def XXH_CPU_LITTLE_ENDIAN * @brief Whether the target is little endian. * * Defined to 1 if the target is little endian, or 0 if it is big endian. * It can be defined externally, for example on the compiler command line. * * If it is not defined, a runtime check (which is usually constant folded) * is used instead. * * @note * This is not necessarily defined to an integer constant. * * @see XXH_isLittleEndian() for the runtime check. */ #ifndef XXH_CPU_LITTLE_ENDIAN /* * Try to detect endianness automatically, to avoid the nonstandard behavior * in `XXH_isLittleEndian()` */ #if defined(_WIN32) /* Windows is always little endian */ \ || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) #define XXH_CPU_LITTLE_ENDIAN 1 #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #define XXH_CPU_LITTLE_ENDIAN 0 #else /*! * @internal * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. * * Most compilers will constant fold this. */ static int XXH_isLittleEndian(void) { /* * Portable and well-defined behavior. * Don't use static: it is detrimental to performance. */ const union { xxh_u32 u; xxh_u8 c[4]; } one = {1}; return one.c[0]; } #define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() #endif #endif /* **************************************** * Compiler-specific Functions and Macros ******************************************/ #define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) #ifdef __has_builtin #define XXH_HAS_BUILTIN(x) __has_builtin(x) #else #define XXH_HAS_BUILTIN(x) 0 #endif /*! * @internal * @def XXH_rotl32(x,r) * @brief 32-bit rotate left. * * @param x The 32-bit integer to be rotated. * @param r The number of bits to rotate. * @pre * @p r > 0 && @p r < 32 * @note * @p x and @p r may be evaluated multiple times. * @return The rotated result. */ #if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) && XXH_HAS_BUILTIN(__builtin_rotateleft64) #define XXH_rotl32 __builtin_rotateleft32 #define XXH_rotl64 __builtin_rotateleft64 /* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ #elif defined(_MSC_VER) #define XXH_rotl32(x, r) _rotl(x, r) #define XXH_rotl64(x, r) _rotl64(x, r) #else #define XXH_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) #define XXH_rotl64(x, r) (((x) << (r)) | ((x) >> (64 - (r)))) #endif /*! * @internal * @fn xxh_u32 XXH_swap32(xxh_u32 x) * @brief A 32-bit byteswap. * * @param x The 32-bit integer to byteswap. * @return @p x, byteswapped. */ #if defined(_MSC_VER) /* Visual Studio */ #define XXH_swap32 _byteswap_ulong #elif XXH_GCC_VERSION >= 403 #define XXH_swap32 __builtin_bswap32 #else static xxh_u32 XXH_swap32(xxh_u32 x) { return ((x << 24) & 0xff000000) | ((x << 8) & 0x00ff0000) | ((x >> 8) & 0x0000ff00) | ((x >> 24) & 0x000000ff); } #endif /* *************************** * Memory reads *****************************/ /*! * @internal * @brief Enum to indicate whether a pointer is aligned. */ typedef enum { XXH_aligned, /*!< Aligned */ XXH_unaligned /*!< Possibly unaligned */ } XXH_alignment; /* * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. * * This is ideal for older compilers which don't inline memcpy. */ #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) { const xxh_u8* bytePtr = (const xxh_u8*)memPtr; return bytePtr[0] | ((xxh_u32)bytePtr[1] << 8) | ((xxh_u32)bytePtr[2] << 16) | ((xxh_u32)bytePtr[3] << 24); } XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) { const xxh_u8* bytePtr = (const xxh_u8*)memPtr; return bytePtr[3] | ((xxh_u32)bytePtr[2] << 8) | ((xxh_u32)bytePtr[1] << 16) | ((xxh_u32)bytePtr[0] << 24); } #else XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); } static xxh_u32 XXH_readBE32(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); } #endif XXH_FORCE_INLINE xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) { if (align == XXH_unaligned) { return XXH_readLE32(ptr); } else { return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); } } /* ************************************* * Misc ***************************************/ /*! @ingroup public */ XXH_PUBLIC_API unsigned XXH_versionNumber(void) { return XXH_VERSION_NUMBER; } /* ******************************************************************* * 32-bit hash functions *********************************************************************/ /*! * @} * @defgroup xxh32_impl XXH32 implementation * @ingroup impl * @{ */ static const xxh_u32 XXH_PRIME32_1 = 0x9E3779B1U; /*!< 0b10011110001101110111100110110001 */ static const xxh_u32 XXH_PRIME32_2 = 0x85EBCA77U; /*!< 0b10000101111010111100101001110111 */ static const xxh_u32 XXH_PRIME32_3 = 0xC2B2AE3DU; /*!< 0b11000010101100101010111000111101 */ static const xxh_u32 XXH_PRIME32_4 = 0x27D4EB2FU; /*!< 0b00100111110101001110101100101111 */ static const xxh_u32 XXH_PRIME32_5 = 0x165667B1U; /*!< 0b00010110010101100110011110110001 */ #ifdef XXH_OLD_NAMES #define PRIME32_1 XXH_PRIME32_1 #define PRIME32_2 XXH_PRIME32_2 #define PRIME32_3 XXH_PRIME32_3 #define PRIME32_4 XXH_PRIME32_4 #define PRIME32_5 XXH_PRIME32_5 #endif /*! * @internal * @brief Normal stripe processing routine. * * This shuffles the bits so that any bit from @p input impacts several bits in * @p acc. * * @param acc The accumulator lane. * @param input The stripe of input to mix. * @return The mixed accumulator lane. */ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { acc += input * XXH_PRIME32_2; acc = XXH_rotl32(acc, 13); acc *= XXH_PRIME32_1; #if defined(__GNUC__) && defined(__SSE4_1__) && !defined(XXH_ENABLE_AUTOVECTORIZE) /* * UGLY HACK: * This inline assembly hack forces acc into a normal register. This is the * only thing that prevents GCC and Clang from autovectorizing the XXH32 * loop (pragmas and attributes don't work for some reason) without globally * disabling SSE4.1. * * The reason we want to avoid vectorization is because despite working on * 4 integers at a time, there are multiple factors slowing XXH32 down on * SSE4: * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on * newer chips!) making it slightly slower to multiply four integers at * once compared to four integers independently. Even when pmulld was * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE * just to multiply unless doing a long operation. * * - Four instructions are required to rotate, * movqda tmp, v // not required with VEX encoding * pslld tmp, 13 // tmp <<= 13 * psrld v, 19 // x >>= 19 * por v, tmp // x |= tmp * compared to one for scalar: * roll v, 13 // reliably fast across the board * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason * * - Instruction level parallelism is actually more beneficial here because * the SIMD actually serializes this operation: While v1 is rotating, v2 * can load data, while v3 can multiply. SSE forces them to operate * together. * * How this hack works: * __asm__("" // Declare an assembly block but don't declare any instructions * : // However, as an Input/Output Operand, * "+r" // constrain a read/write operand (+) as a general purpose register (r). * (acc) // and set acc as the operand * ); * * Because of the 'r', the compiler has promised that seed will be in a * general purpose register and the '+' says that it will be 'read/write', * so it has to assume it has changed. It is like volatile without all the * loads and stores. * * Since the argument has to be in a normal register (not an SSE register), * each time XXH32_round is called, it is impossible to vectorize. */ __asm__("" : "+r"(acc)); #endif return acc; } /*! * @internal * @brief Mixes all bits to finalize the hash. * * The final mix ensures that all input bits have a chance to impact any bit in * the output digest, resulting in an unbiased distribution. * * @param h32 The hash to avalanche. * @return The avalanched hash. */ static xxh_u32 XXH32_avalanche(xxh_u32 h32) { h32 ^= h32 >> 15; h32 *= XXH_PRIME32_2; h32 ^= h32 >> 13; h32 *= XXH_PRIME32_3; h32 ^= h32 >> 16; return (h32); } #define XXH_get32bits(p) XXH_readLE32_align(p, align) /*! * @internal * @brief Processes the last 0-15 bytes of @p ptr. * * There may be up to 15 bytes remaining to consume from the input. * This final stage will digest them to ensure that all input bytes are present * in the final mix. * * @param h32 The hash to finalize. * @param ptr The pointer to the remaining input. * @param len The remaining length, modulo 16. * @param align Whether @p ptr is aligned. * @return The finalized hash. */ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) { #define XXH_PROCESS1 \ do { \ h32 += (*ptr++) * XXH_PRIME32_5; \ h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ } while (0) #define XXH_PROCESS4 \ do { \ h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ ptr += 4; \ h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ } while (0) /* Compact rerolled version */ if (XXH_REROLL) { len &= 15; while (len >= 4) { XXH_PROCESS4; len -= 4; } while (len > 0) { XXH_PROCESS1; --len; } return XXH32_avalanche(h32); } else { switch (len & 15) /* or switch(bEnd - p) */ { case 12: XXH_PROCESS4; /* fallthrough */ case 8: XXH_PROCESS4; /* fallthrough */ case 4: XXH_PROCESS4; return XXH32_avalanche(h32); case 13: XXH_PROCESS4; /* fallthrough */ case 9: XXH_PROCESS4; /* fallthrough */ case 5: XXH_PROCESS4; XXH_PROCESS1; return XXH32_avalanche(h32); case 14: XXH_PROCESS4; /* fallthrough */ case 10: XXH_PROCESS4; /* fallthrough */ case 6: XXH_PROCESS4; XXH_PROCESS1; XXH_PROCESS1; return XXH32_avalanche(h32); case 15: XXH_PROCESS4; /* fallthrough */ case 11: XXH_PROCESS4; /* fallthrough */ case 7: XXH_PROCESS4; /* fallthrough */ case 3: XXH_PROCESS1; /* fallthrough */ case 2: XXH_PROCESS1; /* fallthrough */ case 1: XXH_PROCESS1; /* fallthrough */ case 0: return XXH32_avalanche(h32); } XXH_ASSERT(0); return h32; /* reaching this point is deemed impossible */ } } #ifdef XXH_OLD_NAMES #define PROCESS1 XXH_PROCESS1 #define PROCESS4 XXH_PROCESS4 #else #undef XXH_PROCESS1 #undef XXH_PROCESS4 #endif /*! * @internal * @brief The implementation for @ref XXH32(). * * @param input, len, seed Directly passed from @ref XXH32(). * @param align Whether @p input is aligned. * @return The calculated hash. */ XXH_FORCE_INLINE xxh_u32 XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) { const xxh_u8* bEnd = input + len; xxh_u32 h32; #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) if (input == NULL) { len = 0; bEnd = input = (const xxh_u8*)(size_t)16; } #endif if (len >= 16) { const xxh_u8* const limit = bEnd - 15; xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; xxh_u32 v2 = seed + XXH_PRIME32_2; xxh_u32 v3 = seed + 0; xxh_u32 v4 = seed - XXH_PRIME32_1; do { v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; } while (input < limit); h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); } else { h32 = seed + XXH_PRIME32_5; } h32 += (xxh_u32)len; return XXH32_finalize(h32, input, len & 15, align); } /*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32(const void* input, size_t len, XXH32_hash_t seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH32_state_t state; XXH32_reset(&state, seed); XXH32_update(&state, (const xxh_u8*)input, len); return XXH32_digest(&state); #else if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); } } return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); #endif } /******* Hash streaming *******/ /*! * @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) { return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } /*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } /*! @ingroup xxh32_family */ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) { memcpy(dstState, srcState, sizeof(*dstState)); } /*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) { XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ memset(&state, 0, sizeof(state)); state.v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; state.v2 = seed + XXH_PRIME32_2; state.v3 = seed + 0; state.v4 = seed - XXH_PRIME32_1; /* do not write into reserved, planned to be removed in a future version */ memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); return XXH_OK; } /*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t* state, const void* input, size_t len) { if (input == NULL) #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) return XXH_OK; #else return XXH_ERROR; #endif { const xxh_u8* p = (const xxh_u8*)input; const xxh_u8* const bEnd = p + len; state->total_len_32 += (XXH32_hash_t)len; state->large_len |= (XXH32_hash_t)((len >= 16) | (state->total_len_32 >= 16)); if (state->memsize + len < 16) { /* fill in tmp buffer */ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); state->memsize += (XXH32_hash_t)len; return XXH_OK; } if (state->memsize) { /* some data left from previous update */ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16 - state->memsize); { const xxh_u32* p32 = state->mem32; state->v1 = XXH32_round(state->v1, XXH_readLE32(p32)); p32++; state->v2 = XXH32_round(state->v2, XXH_readLE32(p32)); p32++; state->v3 = XXH32_round(state->v3, XXH_readLE32(p32)); p32++; state->v4 = XXH32_round(state->v4, XXH_readLE32(p32)); } p += 16 - state->memsize; state->memsize = 0; } if (p <= bEnd - 16) { const xxh_u8* const limit = bEnd - 16; xxh_u32 v1 = state->v1; xxh_u32 v2 = state->v2; xxh_u32 v3 = state->v3; xxh_u32 v4 = state->v4; do { v1 = XXH32_round(v1, XXH_readLE32(p)); p += 4; v2 = XXH32_round(v2, XXH_readLE32(p)); p += 4; v3 = XXH32_round(v3, XXH_readLE32(p)); p += 4; v4 = XXH32_round(v4, XXH_readLE32(p)); p += 4; } while (p <= limit); state->v1 = v1; state->v2 = v2; state->v3 = v3; state->v4 = v4; } if (p < bEnd) { XXH_memcpy(state->mem32, p, (size_t)(bEnd - p)); state->memsize = (unsigned)(bEnd - p); } } return XXH_OK; } /*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) { xxh_u32 h32; if (state->large_len) { h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); } else { h32 = state->v3 /* == seed */ + XXH_PRIME32_5; } h32 += state->total_len_32; return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); } /******* Canonical representation *******/ /*! * @ingroup xxh32_family * The default return values from XXH functions are unsigned 32 and 64 bit * integers. * * The canonical representation uses big endian convention, the same convention * as human-readable numbers (large digits first). * * This way, hash values can be written into a file or buffer, remaining * comparable across different systems. * * The following functions allow transformation of hash values to and from their * canonical format. */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); memcpy(dst, &hash, sizeof(*dst)); } /*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) { return XXH_readBE32(src); } #ifndef XXH_NO_LONG_LONG /* ******************************************************************* * 64-bit hash functions *********************************************************************/ /*! * @} * @ingroup impl * @{ */ /******* Memory access *******/ typedef XXH64_hash_t xxh_u64; #ifdef XXH_OLD_NAMES #define U64 xxh_u64 #endif /*! * XXH_REROLL_XXH64: * Whether to reroll the XXH64_finalize() loop. * * Just like XXH32, we can unroll the XXH64_finalize() loop. This can be a * performance gain on 64-bit hosts, as only one jump is required. * * However, on 32-bit hosts, because arithmetic needs to be done with two 32-bit * registers, and 64-bit arithmetic needs to be simulated, it isn't beneficial * to unroll. The code becomes ridiculously large (the largest function in the * binary on i386!), and rerolling it saves anywhere from 3kB to 20kB. It is * also slightly faster because it fits into cache better and is more likely * to be inlined by the compiler. * * If XXH_REROLL is defined, this is ignored and the loop is always rerolled. */ #ifndef XXH_REROLL_XXH64 #if (defined(__ILP32__) || defined(_ILP32)) /* ILP32 is often defined on 32-bit GCC family */ \ || !(defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) /* x86-64 */ \ || defined(_M_ARM64) || defined(__aarch64__) || defined(__arm64__) /* aarch64 */ \ || defined(__PPC64__) || defined(__PPC64LE__) || defined(__ppc64__) || defined(__powerpc64__) /* ppc64 */ \ || defined(__mips64__) || defined(__mips64)) /* mips64 */ \ || (!defined(SIZE_MAX) || SIZE_MAX < ULLONG_MAX) /* check limits */ #define XXH_REROLL_XXH64 1 #else #define XXH_REROLL_XXH64 0 #endif #endif /* !defined(XXH_REROLL_XXH64) */ #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) /* * Manual byteshift. Best for old compilers which don't inline memcpy. * We actually directly use XXH_readLE64 and XXH_readBE64. */ #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 2)) /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ static xxh_u64 XXH_read64(const void* memPtr) { return *(const xxh_u64*)memPtr; } #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 1)) /* * __pack instructions are safer, but compiler specific, hence potentially * problematic for some compilers. * * Currently only defined for GCC and ICC. */ #ifdef XXH_OLD_NAMES typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; #endif static xxh_u64 XXH_read64(const void* ptr) { typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64; return ((const xxh_unalign64*)ptr)->u64; } #else /* * Portable and safe solution. Generally efficient. * see: https://stackoverflow.com/a/32095106/646947 */ static xxh_u64 XXH_read64(const void* memPtr) { xxh_u64 val; memcpy(&val, memPtr, sizeof(val)); return val; } #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ #if defined(_MSC_VER) /* Visual Studio */ #define XXH_swap64 _byteswap_uint64 #elif XXH_GCC_VERSION >= 403 #define XXH_swap64 __builtin_bswap64 #else static xxh_u64 XXH_swap64(xxh_u64 x) { return ((x << 56) & 0xff00000000000000ULL) | ((x << 40) & 0x00ff000000000000ULL) | ((x << 24) & 0x0000ff0000000000ULL) | ((x << 8) & 0x000000ff00000000ULL) | ((x >> 8) & 0x00000000ff000000ULL) | ((x >> 24) & 0x0000000000ff0000ULL) | ((x >> 40) & 0x000000000000ff00ULL) | ((x >> 56) & 0x00000000000000ffULL); } #endif /* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) { const xxh_u8* bytePtr = (const xxh_u8*)memPtr; return bytePtr[0] | ((xxh_u64)bytePtr[1] << 8) | ((xxh_u64)bytePtr[2] << 16) | ((xxh_u64)bytePtr[3] << 24) | ((xxh_u64)bytePtr[4] << 32) | ((xxh_u64)bytePtr[5] << 40) | ((xxh_u64)bytePtr[6] << 48) | ((xxh_u64)bytePtr[7] << 56); } XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) { const xxh_u8* bytePtr = (const xxh_u8*)memPtr; return bytePtr[7] | ((xxh_u64)bytePtr[6] << 8) | ((xxh_u64)bytePtr[5] << 16) | ((xxh_u64)bytePtr[4] << 24) | ((xxh_u64)bytePtr[3] << 32) | ((xxh_u64)bytePtr[2] << 40) | ((xxh_u64)bytePtr[1] << 48) | ((xxh_u64)bytePtr[0] << 56); } #else XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); } static xxh_u64 XXH_readBE64(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); } #endif XXH_FORCE_INLINE xxh_u64 XXH_readLE64_align(const void* ptr, XXH_alignment align) { if (align == XXH_unaligned) return XXH_readLE64(ptr); else return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); } /******* xxh64 *******/ /*! * @} * @defgroup xxh64_impl XXH64 implementation * @ingroup impl * @{ */ static const xxh_u64 XXH_PRIME64_1 = 0x9E3779B185EBCA87ULL; /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ static const xxh_u64 XXH_PRIME64_2 = 0xC2B2AE3D27D4EB4FULL; /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ static const xxh_u64 XXH_PRIME64_3 = 0x165667B19E3779F9ULL; /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ static const xxh_u64 XXH_PRIME64_4 = 0x85EBCA77C2B2AE63ULL; /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ static const xxh_u64 XXH_PRIME64_5 = 0x27D4EB2F165667C5ULL; /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ #ifdef XXH_OLD_NAMES #define PRIME64_1 XXH_PRIME64_1 #define PRIME64_2 XXH_PRIME64_2 #define PRIME64_3 XXH_PRIME64_3 #define PRIME64_4 XXH_PRIME64_4 #define PRIME64_5 XXH_PRIME64_5 #endif static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) { acc += input * XXH_PRIME64_2; acc = XXH_rotl64(acc, 31); acc *= XXH_PRIME64_1; return acc; } static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) { val = XXH64_round(0, val); acc ^= val; acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; return acc; } static xxh_u64 XXH64_avalanche(xxh_u64 h64) { h64 ^= h64 >> 33; h64 *= XXH_PRIME64_2; h64 ^= h64 >> 29; h64 *= XXH_PRIME64_3; h64 ^= h64 >> 32; return h64; } #define XXH_get64bits(p) XXH_readLE64_align(p, align) static xxh_u64 XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) { #define XXH_PROCESS1_64 \ do { \ h64 ^= (*ptr++) * XXH_PRIME64_5; \ h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; \ } while (0) #define XXH_PROCESS4_64 \ do { \ h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; \ ptr += 4; \ h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; \ } while (0) #define XXH_PROCESS8_64 \ do { \ xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); \ ptr += 8; \ h64 ^= k1; \ h64 = XXH_rotl64(h64, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; \ } while (0) /* Rerolled version for 32-bit targets is faster and much smaller. */ if (XXH_REROLL || XXH_REROLL_XXH64) { len &= 31; while (len >= 8) { XXH_PROCESS8_64; len -= 8; } if (len >= 4) { XXH_PROCESS4_64; len -= 4; } while (len > 0) { XXH_PROCESS1_64; --len; } return XXH64_avalanche(h64); } else { switch (len & 31) { case 24: XXH_PROCESS8_64; /* fallthrough */ case 16: XXH_PROCESS8_64; /* fallthrough */ case 8: XXH_PROCESS8_64; return XXH64_avalanche(h64); case 28: XXH_PROCESS8_64; /* fallthrough */ case 20: XXH_PROCESS8_64; /* fallthrough */ case 12: XXH_PROCESS8_64; /* fallthrough */ case 4: XXH_PROCESS4_64; return XXH64_avalanche(h64); case 25: XXH_PROCESS8_64; /* fallthrough */ case 17: XXH_PROCESS8_64; /* fallthrough */ case 9: XXH_PROCESS8_64; XXH_PROCESS1_64; return XXH64_avalanche(h64); case 29: XXH_PROCESS8_64; /* fallthrough */ case 21: XXH_PROCESS8_64; /* fallthrough */ case 13: XXH_PROCESS8_64; /* fallthrough */ case 5: XXH_PROCESS4_64; XXH_PROCESS1_64; return XXH64_avalanche(h64); case 26: XXH_PROCESS8_64; /* fallthrough */ case 18: XXH_PROCESS8_64; /* fallthrough */ case 10: XXH_PROCESS8_64; XXH_PROCESS1_64; XXH_PROCESS1_64; return XXH64_avalanche(h64); case 30: XXH_PROCESS8_64; /* fallthrough */ case 22: XXH_PROCESS8_64; /* fallthrough */ case 14: XXH_PROCESS8_64; /* fallthrough */ case 6: XXH_PROCESS4_64; XXH_PROCESS1_64; XXH_PROCESS1_64; return XXH64_avalanche(h64); case 27: XXH_PROCESS8_64; /* fallthrough */ case 19: XXH_PROCESS8_64; /* fallthrough */ case 11: XXH_PROCESS8_64; XXH_PROCESS1_64; XXH_PROCESS1_64; XXH_PROCESS1_64; return XXH64_avalanche(h64); case 31: XXH_PROCESS8_64; /* fallthrough */ case 23: XXH_PROCESS8_64; /* fallthrough */ case 15: XXH_PROCESS8_64; /* fallthrough */ case 7: XXH_PROCESS4_64; /* fallthrough */ case 3: XXH_PROCESS1_64; /* fallthrough */ case 2: XXH_PROCESS1_64; /* fallthrough */ case 1: XXH_PROCESS1_64; /* fallthrough */ case 0: return XXH64_avalanche(h64); } } /* impossible to reach */ XXH_ASSERT(0); return 0; /* unreachable, but some compilers complain without it */ } #ifdef XXH_OLD_NAMES #define PROCESS1_64 XXH_PROCESS1_64 #define PROCESS4_64 XXH_PROCESS4_64 #define PROCESS8_64 XXH_PROCESS8_64 #else #undef XXH_PROCESS1_64 #undef XXH_PROCESS4_64 #undef XXH_PROCESS8_64 #endif XXH_FORCE_INLINE xxh_u64 XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) { const xxh_u8* bEnd = input + len; xxh_u64 h64; #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) if (input == NULL) { len = 0; bEnd = input = (const xxh_u8*)(size_t)32; } #endif if (len >= 32) { const xxh_u8* const limit = bEnd - 32; xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; xxh_u64 v2 = seed + XXH_PRIME64_2; xxh_u64 v3 = seed + 0; xxh_u64 v4 = seed - XXH_PRIME64_1; do { v1 = XXH64_round(v1, XXH_get64bits(input)); input += 8; v2 = XXH64_round(v2, XXH_get64bits(input)); input += 8; v3 = XXH64_round(v3, XXH_get64bits(input)); input += 8; v4 = XXH64_round(v4, XXH_get64bits(input)); input += 8; } while (input <= limit); h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); h64 = XXH64_mergeRound(h64, v1); h64 = XXH64_mergeRound(h64, v2); h64 = XXH64_mergeRound(h64, v3); h64 = XXH64_mergeRound(h64, v4); } else { h64 = seed + XXH_PRIME64_5; } h64 += (xxh_u64)len; return XXH64_finalize(h64, input, len, align); } /*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t len, XXH64_hash_t seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH64_state_t state; XXH64_reset(&state, seed); XXH64_update(&state, (const xxh_u8*)input, len); return XXH64_digest(&state); #else if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 7) == 0) { /* Input is aligned, let's leverage the speed advantage */ return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); } } return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); #endif } /******* Hash Streaming *******/ /*! @ingroup xxh64_family*/ XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) { return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } /*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } /*! @ingroup xxh64_family */ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) { memcpy(dstState, srcState, sizeof(*dstState)); } /*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed) { XXH64_state_t state; /* use a local state to memcpy() in order to avoid strict-aliasing warnings */ memset(&state, 0, sizeof(state)); state.v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; state.v2 = seed + XXH_PRIME64_2; state.v3 = seed + 0; state.v4 = seed - XXH_PRIME64_1; /* do not write into reserved64, might be removed in a future version */ memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved64)); return XXH_OK; } /*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t* state, const void* input, size_t len) { if (input == NULL) #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) return XXH_OK; #else return XXH_ERROR; #endif { const xxh_u8* p = (const xxh_u8*)input; const xxh_u8* const bEnd = p + len; state->total_len += len; if (state->memsize + len < 32) { /* fill in tmp buffer */ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); state->memsize += (xxh_u32)len; return XXH_OK; } if (state->memsize) { /* tmp buffer is full */ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32 - state->memsize); state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64 + 0)); state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64 + 1)); state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64 + 2)); state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64 + 3)); p += 32 - state->memsize; state->memsize = 0; } if (p + 32 <= bEnd) { const xxh_u8* const limit = bEnd - 32; xxh_u64 v1 = state->v1; xxh_u64 v2 = state->v2; xxh_u64 v3 = state->v3; xxh_u64 v4 = state->v4; do { v1 = XXH64_round(v1, XXH_readLE64(p)); p += 8; v2 = XXH64_round(v2, XXH_readLE64(p)); p += 8; v3 = XXH64_round(v3, XXH_readLE64(p)); p += 8; v4 = XXH64_round(v4, XXH_readLE64(p)); p += 8; } while (p <= limit); state->v1 = v1; state->v2 = v2; state->v3 = v3; state->v4 = v4; } if (p < bEnd) { XXH_memcpy(state->mem64, p, (size_t)(bEnd - p)); state->memsize = (unsigned)(bEnd - p); } } return XXH_OK; } /*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state) { xxh_u64 h64; if (state->total_len >= 32) { xxh_u64 const v1 = state->v1; xxh_u64 const v2 = state->v2; xxh_u64 const v3 = state->v3; xxh_u64 const v4 = state->v4; h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); h64 = XXH64_mergeRound(h64, v1); h64 = XXH64_mergeRound(h64, v2); h64 = XXH64_mergeRound(h64, v3); h64 = XXH64_mergeRound(h64, v4); } else { h64 = state->v3 /*seed*/ + XXH_PRIME64_5; } h64 += (xxh_u64)state->total_len; return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); } /******* Canonical representation *******/ /*! @ingroup xxh64_family */ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); memcpy(dst, &hash, sizeof(*dst)); } /*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) { return XXH_readBE64(src); } /* ********************************************************************* * XXH3 * New generation hash designed for speed on small keys and vectorization ************************************************************************ */ /*! * @} * @defgroup xxh3_impl XXH3 implementation * @ingroup impl * @{ */ /* === Compiler specifics === */ #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ #define XXH_RESTRICT restrict #else /* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ #define XXH_RESTRICT /* disable */ #endif #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || \ defined(__clang__) #define XXH_likely(x) __builtin_expect(x, 1) #define XXH_unlikely(x) __builtin_expect(x, 0) #else #define XXH_likely(x) (x) #define XXH_unlikely(x) (x) #endif #if defined(__GNUC__) #if defined(__AVX2__) #include #elif defined(__SSE2__) #include #elif defined(__ARM_NEON__) || defined(__ARM_NEON) #define inline __inline__ /* circumvent a clang bug */ #include #undef inline #endif #elif defined(_MSC_VER) #include #endif /* * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while * remaining a true 64-bit/128-bit hash function. * * This is done by prioritizing a subset of 64-bit operations that can be * emulated without too many steps on the average 32-bit machine. * * For example, these two lines seem similar, and run equally fast on 64-bit: * * xxh_u64 x; * x ^= (x >> 47); // good * x ^= (x >> 13); // bad * * However, to a 32-bit machine, there is a major difference. * * x ^= (x >> 47) looks like this: * * x.lo ^= (x.hi >> (47 - 32)); * * while x ^= (x >> 13) looks like this: * * // note: funnel shifts are not usually cheap. * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); * x.hi ^= (x.hi >> 13); * * The first one is significantly faster than the second, simply because the * shift is larger than 32. This means: * - All the bits we need are in the upper 32 bits, so we can ignore the lower * 32 bits in the shift. * - The shift result will always fit in the lower 32 bits, and therefore, * we can ignore the upper 32 bits in the xor. * * Thanks to this optimization, XXH3 only requires these features to be efficient: * * - Usable unaligned access * - A 32-bit or 64-bit ALU * - If 32-bit, a decent ADC instruction * - A 32 or 64-bit multiply with a 64-bit result * - For the 128-bit variant, a decent byteswap helps short inputs. * * The first two are already required by XXH32, and almost all 32-bit and 64-bit * platforms which can run XXH32 can run XXH3 efficiently. * * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one * notable exception. * * First of all, Thumb-1 lacks support for the UMULL instruction which * performs the important long multiply. This means numerous __aeabi_lmul * calls. * * Second of all, the 8 functional registers are just not enough. * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need * Lo registers, and this shuffling results in thousands more MOVs than A32. * * A32 and T32 don't have this limitation. They can access all 14 registers, * do a 32->64 multiply with UMULL, and the flexible operand allowing free * shifts is helpful, too. * * Therefore, we do a quick sanity check. * * If compiling Thumb-1 for a target which supports ARM instructions, we will * emit a warning, as it is not a "sane" platform to compile for. * * Usually, if this happens, it is because of an accident and you probably need * to specify -march, as you likely meant to compile for a newer architecture. * * Credit: large sections of the vectorial and asm source code paths * have been contributed by @easyaspi314 */ #if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) #warning "XXH3 is highly inefficient without ARM or Thumb-2." #endif /* ========================================== * Vectorization detection * ========================================== */ #ifdef XXH_DOXYGEN /*! * @ingroup tuning * @brief Overrides the vectorization implementation chosen for XXH3. * * Can be defined to 0 to disable SIMD or any of the values mentioned in * @ref XXH_VECTOR_TYPE. * * If this is not defined, it uses predefined macros to determine the best * implementation. */ #define XXH_VECTOR XXH_SCALAR /*! * @ingroup tuning * @brief Possible values for @ref XXH_VECTOR. * * Note that these are actually implemented as macros. * * If this is not defined, it is detected automatically. * @ref XXH_X86DISPATCH overrides this. */ enum XXH_VECTOR_TYPE /* fake enum */ { XXH_SCALAR = 0, /*!< Portable scalar version */ XXH_SSE2 = 1, /*!< * SSE2 for Pentium 4, Opteron, all x86_64. * * @note SSE2 is also guaranteed on Windows 10, macOS, and * Android x86. */ XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ }; /*! * @ingroup tuning * @brief Selects the minimum alignment for XXH3's accumulators. * * When using SIMD, this should match the alignment reqired for said vector * type, so, for example, 32 for AVX2. * * Default: Auto detected. */ #define XXH_ACC_ALIGN 8 #endif /* Actual definition */ #ifndef XXH_DOXYGEN #define XXH_SCALAR 0 #define XXH_SSE2 1 #define XXH_AVX2 2 #define XXH_AVX512 3 #define XXH_NEON 4 #define XXH_VSX 5 #endif #ifndef XXH_VECTOR /* can be defined on command line */ #if defined(__AVX512F__) #define XXH_VECTOR XXH_AVX512 #elif defined(__AVX2__) #define XXH_VECTOR XXH_AVX2 #elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) #define XXH_VECTOR XXH_SSE2 #elif defined(__GNUC__) /* msvc support maybe later */ \ && (defined(__ARM_NEON__) || defined(__ARM_NEON)) && \ (defined(__LITTLE_ENDIAN__) /* We only support little endian NEON */ \ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) #define XXH_VECTOR XXH_NEON #elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) || \ (defined(__s390x__) && defined(__VEC__)) && defined(__GNUC__) /* TODO: IBM XL */ #define XXH_VECTOR XXH_VSX #else #define XXH_VECTOR XXH_SCALAR #endif #endif /* * Controls the alignment of the accumulator, * for compatibility with aligned vector loads, which are usually faster. */ #ifndef XXH_ACC_ALIGN #if defined(XXH_X86DISPATCH) #define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ #elif XXH_VECTOR == XXH_SCALAR /* scalar */ #define XXH_ACC_ALIGN 8 #elif XXH_VECTOR == XXH_SSE2 /* sse2 */ #define XXH_ACC_ALIGN 16 #elif XXH_VECTOR == XXH_AVX2 /* avx2 */ #define XXH_ACC_ALIGN 32 #elif XXH_VECTOR == XXH_NEON /* neon */ #define XXH_ACC_ALIGN 16 #elif XXH_VECTOR == XXH_VSX /* vsx */ #define XXH_ACC_ALIGN 16 #elif XXH_VECTOR == XXH_AVX512 /* avx512 */ #define XXH_ACC_ALIGN 64 #endif #endif #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 #define XXH_SEC_ALIGN XXH_ACC_ALIGN #else #define XXH_SEC_ALIGN 8 #endif /* * UGLY HACK: * GCC usually generates the best code with -O3 for xxHash. * * However, when targeting AVX2, it is overzealous in its unrolling resulting * in code roughly 3/4 the speed of Clang. * * There are other issues, such as GCC splitting _mm256_loadu_si256 into * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which * only applies to Sandy and Ivy Bridge... which don't even support AVX2. * * That is why when compiling the AVX2 version, it is recommended to use either * -O2 -mavx2 -march=haswell * or * -O2 -mavx2 -mno-avx256-split-unaligned-load * for decent performance, or to use Clang instead. * * Fortunately, we can control the first one with a pragma that forces GCC into * -O2, but the other one we can't control without "failed to inline always * inline function due to target mismatch" warnings. */ #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ #pragma GCC push_options #pragma GCC optimize("-O2") #endif #if XXH_VECTOR == XXH_NEON /* * NEON's setup for vmlal_u32 is a little more complicated than it is on * SSE2, AVX2, and VSX. * * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. * * To do the same operation, the 128-bit 'Q' register needs to be split into * two 64-bit 'D' registers, performing this operation:: * * [ a | b ] * | '---------. .--------' | * | x | * | .---------' '--------. | * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] * * Due to significant changes in aarch64, the fastest method for aarch64 is * completely different than the fastest method for ARMv7-A. * * ARMv7-A treats D registers as unions overlaying Q registers, so modifying * D11 will modify the high half of Q5. This is similar to how modifying AH * will only affect bits 8-15 of AX on x86. * * VZIP takes two registers, and puts even lanes in one register and odd lanes * in the other. * * On ARMv7-A, this strangely modifies both parameters in place instead of * taking the usual 3-operand form. * * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the * lower and upper halves of the Q register to end up with the high and low * halves where we want - all in one instruction. * * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } * * Unfortunately we need inline assembly for this: Instructions modifying two * registers at once is not possible in GCC or Clang's IR, and they have to * create a copy. * * aarch64 requires a different approach. * * In order to make it easier to write a decent compiler for aarch64, many * quirks were removed, such as conditional execution. * * NEON was also affected by this. * * aarch64 cannot access the high bits of a Q-form register, and writes to a * D-form register zero the high bits, similar to how writes to W-form scalar * registers (or DWORD registers on x86_64) work. * * The formerly free vget_high intrinsics now require a vext (with a few * exceptions) * * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one * operand. * * The equivalent of the VZIP.32 on the lower and upper halves would be this * mess: * * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } * * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): * * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); * * This is available on ARMv7-A, but is less efficient than a single VZIP.32. */ /*! * Function-like macro: * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) * { * outLo = (uint32x2_t)(in & 0xFFFFFFFF); * outHi = (uint32x2_t)(in >> 32); * in = UNDEFINED; * } */ #if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ && defined(__GNUC__) && !defined(__aarch64__) && !defined(__arm64__) #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ do { \ /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ __asm__("vzip.32 %e0, %f0" : "+w"(in)); \ (outLo) = vget_low_u32(vreinterpretq_u32_u64(in)); \ (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ } while (0) #else #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ do { \ (outLo) = vmovn_u64(in); \ (outHi) = vshrn_n_u64((in), 32); \ } while (0) #endif #endif /* XXH_VECTOR == XXH_NEON */ /* * VSX and Z Vector helpers. * * This is very messy, and any pull requests to clean this up are welcome. * * There are a lot of problems with supporting VSX and s390x, due to * inconsistent intrinsics, spotty coverage, and multiple endiannesses. */ #if XXH_VECTOR == XXH_VSX #if defined(__s390x__) #include #else /* gcc's altivec.h can have the unwanted consequence to unconditionally * #define bool, vector, and pixel keywords, * with bad consequences for programs already using these keywords for other purposes. * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined. * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler, * but it seems that, in some cases, it isn't. * Force the build macro to be defined, so that keywords are not altered. */ #if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) #define __APPLE_ALTIVEC__ #endif #include #endif typedef __vector unsigned long long xxh_u64x2; typedef __vector unsigned char xxh_u8x16; typedef __vector unsigned xxh_u32x4; #ifndef XXH_VSX_BE #if defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #define XXH_VSX_BE 1 #elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ #warning "-maltivec=be is not recommended. Please use native endianness." #define XXH_VSX_BE 1 #else #define XXH_VSX_BE 0 #endif #endif /* !defined(XXH_VSX_BE) */ #if XXH_VSX_BE #if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) #define XXH_vec_revb vec_revb #else /*! * A polyfill for POWER9's vec_revb(). */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) { xxh_u8x16 const vByteSwap = {0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08}; return vec_perm(val, val, vByteSwap); } #endif #endif /* XXH_VSX_BE */ /*! * Performs an unaligned vector load and byte swaps it on big endian. */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void* ptr) { xxh_u64x2 ret; memcpy(&ret, ptr, sizeof(xxh_u64x2)); #if XXH_VSX_BE ret = XXH_vec_revb(ret); #endif return ret; } /* * vec_mulo and vec_mule are very problematic intrinsics on PowerPC * * These intrinsics weren't added until GCC 8, despite existing for a while, * and they are endian dependent. Also, their meaning swap depending on version. * */ #if defined(__s390x__) /* s390x is always big endian, no issue on this platform */ #define XXH_vec_mulo vec_mulo #define XXH_vec_mule vec_mule #elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) /* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ #define XXH_vec_mulo __builtin_altivec_vmulouw #define XXH_vec_mule __builtin_altivec_vmuleuw #else /* gcc needs inline assembly */ /* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) { xxh_u64x2 result; __asm__("vmulouw %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); return result; } XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) { xxh_u64x2 result; __asm__("vmuleuw %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); return result; } #endif /* XXH_vec_mulo, XXH_vec_mule */ #endif /* XXH_VECTOR == XXH_VSX */ /* prefetch * can be disabled, by declaring XXH_NO_PREFETCH build macro */ #if defined(XXH_NO_PREFETCH) #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ #else #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ #include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ #define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) #elif defined(__GNUC__) && ((__GNUC__ >= 4) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))) #define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) #else #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ #endif #endif /* XXH_NO_PREFETCH */ /* ========================================== * XXH3 default settings * ========================================== */ #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) #error "default keyset is not large enough" #endif /*! Pseudorandom secret taken directly from FARSH. */ XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, }; #ifdef XXH_OLD_NAMES #define kSecret XXH3_kSecret #endif #ifdef XXH_DOXYGEN /*! * @brief Calculates a 32-bit to 64-bit long multiply. * * Implemented as a macro. * * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't * need to (but it shouldn't need to anyways, it is about 7 instructions to do * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we * use that instead of the normal method. * * If you are compiling for platforms like Thumb-1 and don't have a better option, * you may also want to write your own long multiply routine here. * * @param x, y Numbers to be multiplied * @return 64-bit product of the low 32 bits of @p x and @p y. */ XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); } #elif defined(_MSC_VER) && defined(_M_IX86) #include #define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) #else /* * Downcast + upcast is usually better than masking on older compilers like * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. * * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands * and perform a full 64x64 multiply -- entirely redundant on 32-bit. */ #define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) #endif /*! * @brief Calculates a 64->128-bit long multiply. * * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar * version. * * @param lhs, rhs The 64-bit integers to be multiplied * @return The 128-bit result represented in an @ref XXH128_hash_t. */ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { /* * GCC/Clang __uint128_t method. * * On most 64-bit targets, GCC and Clang define a __uint128_t type. * This is usually the best way as it usually uses a native long 64-bit * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. * * Usually. * * Despite being a 32-bit platform, Clang (and emscripten) define this type * despite not having the arithmetic for it. This results in a laggy * compiler builtin call which calculates a full 128-bit multiply. * In that case it is best to use the portable one. * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 */ #if defined(__GNUC__) && !defined(__wasm__) && defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; XXH128_hash_t r128; r128.low64 = (xxh_u64)(product); r128.high64 = (xxh_u64)(product >> 64); return r128; /* * MSVC for x64's _umul128 method. * * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); * * This compiles to single operand MUL on x64. */ #elif defined(_M_X64) || defined(_M_IA64) #ifndef _MSC_VER #pragma intrinsic(_umul128) #endif xxh_u64 product_high; xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); XXH128_hash_t r128; r128.low64 = product_low; r128.high64 = product_high; return r128; #else /* * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. * * This is a fast and simple grade school multiply, which is shown below * with base 10 arithmetic instead of base 0x100000000. * * 9 3 // D2 lhs = 93 * x 7 5 // D2 rhs = 75 * ---------- * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 * --------- * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 * --------- * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 * * The reasons for adding the products like this are: * 1. It avoids manual carry tracking. Just like how * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. * This avoids a lot of complexity. * * 2. It hints for, and on Clang, compiles to, the powerful UMAAL * instruction available in ARM's Digital Signal Processing extension * in 32-bit ARMv6 and later, which is shown below: * * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) * { * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); * *RdHi = (xxh_u32)(product >> 32); * } * * This instruction was designed for efficient long multiplication, and * allows this to be calculated in only 4 instructions at speeds * comparable to some 64-bit ALUs. * * 3. It isn't terrible on other platforms. Usually this will be a couple * of 32-bit ADD/ADCs. */ /* First calculate all of the cross products. */ xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); /* Now add the products together. These will never overflow. */ xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); XXH128_hash_t r128; r128.low64 = lower; r128.high64 = upper; return r128; #endif } /*! * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. * * The reason for the separate function is to prevent passing too many structs * around by value. This will hopefully inline the multiply, but we don't force it. * * @param lhs, rhs The 64-bit integers to multiply * @return The low 64 bits of the product XOR'd by the high 64 bits. * @see XXH_mult64to128() */ static xxh_u64 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { XXH128_hash_t product = XXH_mult64to128(lhs, rhs); return product.low64 ^ product.high64; } /*! Seems to produce slightly better code on GCC for some reason. */ XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) { XXH_ASSERT(0 <= shift && shift < 64); return v64 ^ (v64 >> shift); } /* * This is a fast avalanche stage, * suitable when input bits are already partially mixed */ static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) { h64 = XXH_xorshift64(h64, 37); h64 *= 0x165667919E3779F9ULL; h64 = XXH_xorshift64(h64, 32); return h64; } /* * This is a stronger avalanche, * inspired by Pelle Evensen's rrmxmx * preferable when input has not been previously mixed */ static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) { /* this mix is inspired by Pelle Evensen's rrmxmx */ h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); h64 *= 0x9FB21C651E98DF25ULL; h64 ^= (h64 >> 35) + len; h64 *= 0x9FB21C651E98DF25ULL; return XXH_xorshift64(h64, 28); } /* ========================================== * Short keys * ========================================== * One of the shortcomings of XXH32 and XXH64 was that their performance was * sub-optimal on short lengths. It used an iterative algorithm which strongly * favored lengths that were a multiple of 4 or 8. * * Instead of iterating over individual inputs, we use a set of single shot * functions which piece together a range of lengths and operate in constant time. * * Additionally, the number of multiplies has been significantly reduced. This * reduces latency, especially when emulating 64-bit multiplies on 32-bit. * * Depending on the platform, this may or may not be faster than XXH32, but it * is almost guaranteed to be faster than XXH64. */ /* * At very short lengths, there isn't enough input to fully hide secrets, or use * the entire secret. * * There is also only a limited amount of mixing we can do before significantly * impacting performance. * * Therefore, we use different sections of the secret and always mix two secret * samples with an XOR. This should have no effect on performance on the * seedless or withSeed variants because everything _should_ be constant folded * by modern compilers. * * The XOR mixing hides individual parts of the secret and increases entropy. * * This adds an extra layer of strength for custom secrets. */ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(1 <= len && len <= 3); XXH_ASSERT(secret != NULL); /* * len = 1: combined = { input[0], 0x01, input[0], input[0] } * len = 2: combined = { input[1], 0x02, input[0], input[1] } * len = 3: combined = { input[2], 0x03, input[0], input[1] } */ { xxh_u8 const c1 = input[0]; xxh_u8 const c2 = input[len >> 1]; xxh_u8 const c3 = input[len - 1]; xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret + 4)) + seed; xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; return XXH64_avalanche(keyed); } } XXH_FORCE_INLINE XXH64_hash_t XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); XXH_ASSERT(4 <= len && len < 8); seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; { xxh_u32 const input1 = XXH_readLE32(input); xxh_u32 const input2 = XXH_readLE32(input + len - 4); xxh_u64 const bitflip = (XXH_readLE64(secret + 8) ^ XXH_readLE64(secret + 16)) - seed; xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); xxh_u64 const keyed = input64 ^ bitflip; return XXH3_rrmxmx(keyed, len); } } XXH_FORCE_INLINE XXH64_hash_t XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); XXH_ASSERT(8 <= len && len <= 16); { xxh_u64 const bitflip1 = (XXH_readLE64(secret + 24) ^ XXH_readLE64(secret + 32)) + seed; xxh_u64 const bitflip2 = (XXH_readLE64(secret + 40) ^ XXH_readLE64(secret + 48)) - seed; xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; xxh_u64 const acc = len + XXH_swap64(input_lo) + input_hi + XXH3_mul128_fold64(input_lo, input_hi); return XXH3_avalanche(acc); } } XXH_FORCE_INLINE XXH64_hash_t XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(len <= 16); { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); if (len) return XXH3_len_1to3_64b(input, len, secret, seed); return XXH64_avalanche(seed ^ (XXH_readLE64(secret + 56) ^ XXH_readLE64(secret + 64))); } } /* * DISCLAIMER: There are known *seed-dependent* multicollisions here due to * multiplication by zero, affecting hashes of lengths 17 to 240. * * However, they are very unlikely. * * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all * unseeded non-cryptographic hashes, it does not attempt to defend itself * against specially crafted inputs, only random inputs. * * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes * cancelling out the secret is taken an arbitrary number of times (addressed * in XXH3_accumulate_512), this collision is very unlikely with random inputs * and/or proper seeding: * * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a * function that is only called up to 16 times per hash with up to 240 bytes of * input. * * This is not too bad for a non-cryptographic hash function, especially with * only 64 bit outputs. * * The 128-bit variant (which trades some speed for strength) is NOT affected * by this, although it is always a good idea to use a proper seed if you care * about strength. */ XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) { #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ /* * UGLY HACK: * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in * slower code. * * By forcing seed64 into a register, we disrupt the cost model and * cause it to scalarize. See `XXH32_round()` * * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on * GCC 9.2, despite both emitting scalar code. * * GCC generates much better scalar code than Clang for the rest of XXH3, * which is why finding a more optimal codepath is an interest. */ __asm__("" : "+r"(seed64)); #endif { xxh_u64 const input_lo = XXH_readLE64(input); xxh_u64 const input_hi = XXH_readLE64(input + 8); return XXH3_mul128_fold64(input_lo ^ (XXH_readLE64(secret) + seed64), input_hi ^ (XXH_readLE64(secret + 8) - seed64)); } } /* For mid range keys, XXH3 uses a Mum-hash variant. */ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; XXH_ASSERT(16 < len && len <= 128); { xxh_u64 acc = len * XXH_PRIME64_1; if (len > 32) { if (len > 64) { if (len > 96) { acc += XXH3_mix16B(input + 48, secret + 96, seed); acc += XXH3_mix16B(input + len - 64, secret + 112, seed); } acc += XXH3_mix16B(input + 32, secret + 64, seed); acc += XXH3_mix16B(input + len - 48, secret + 80, seed); } acc += XXH3_mix16B(input + 16, secret + 32, seed); acc += XXH3_mix16B(input + len - 32, secret + 48, seed); } acc += XXH3_mix16B(input + 0, secret + 0, seed); acc += XXH3_mix16B(input + len - 16, secret + 16, seed); return XXH3_avalanche(acc); } } #define XXH3_MIDSIZE_MAX 240 XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); #define XXH3_MIDSIZE_STARTOFFSET 3 #define XXH3_MIDSIZE_LASTOFFSET 17 { xxh_u64 acc = len * XXH_PRIME64_1; int const nbRounds = (int)len / 16; int i; for (i = 0; i < 8; i++) { acc += XXH3_mix16B(input + (16 * i), secret + (16 * i), seed); } acc = XXH3_avalanche(acc); XXH_ASSERT(nbRounds >= 8); #if defined(__clang__) /* Clang */ \ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ /* * UGLY HACK: * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. * In everywhere else, it uses scalar code. * * For 64->128-bit multiplies, even if the NEON was 100% optimal, it * would still be slower than UMAAL (see XXH_mult64to128). * * Unfortunately, Clang doesn't handle the long multiplies properly and * converts them to the nonexistent "vmulq_u64" intrinsic, which is then * scalarized into an ugly mess of VMOV.32 instructions. * * This mess is difficult to avoid without turning autovectorization * off completely, but they are usually relatively minor and/or not * worth it to fix. * * This loop is the easiest to fix, as unlike XXH32, this pragma * _actually works_ because it is a loop vectorization instead of an * SLP vectorization. */ #pragma clang loop vectorize(disable) #endif for (i = 8; i < nbRounds; i++) { acc += XXH3_mix16B(input + (16 * i), secret + (16 * (i - 8)) + XXH3_MIDSIZE_STARTOFFSET, seed); } /* last bytes */ acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); return XXH3_avalanche(acc); } } /* ======= Long Keys ======= */ #define XXH_STRIPE_LEN 64 #define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) #ifdef XXH_OLD_NAMES #define STRIPE_LEN XXH_STRIPE_LEN #define ACC_NB XXH_ACC_NB #endif XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) { if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); memcpy(dst, &v64, sizeof(v64)); } /* Several intrinsic functions below are supposed to accept __int64 as argument, * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . * However, several environments do not define __int64 type, * requiring a workaround. */ #if !defined(__VMS) && (defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 \ */)) typedef int64_t xxh_i64; #else /* the following type must have a width of 64-bit */ typedef long long xxh_i64; #endif /* * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. * * It is a hardened version of UMAC, based off of FARSH's implementation. * * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD * implementations, and it is ridiculously fast. * * We harden it by mixing the original input to the accumulators as well as the product. * * This means that in the (relatively likely) case of a multiply by zero, the * original input is preserved. * * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve * cross-pollination, as otherwise the upper and lower halves would be * essentially independent. * * This doesn't matter on 64-bit hashes since they all get merged together in * the end, so we skip the extra step. * * Both XXH3_64bits and XXH3_128bits use this subroutine. */ #if (XXH_VECTOR == XXH_AVX512) || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) #ifndef XXH_TARGET_AVX512 #define XXH_TARGET_AVX512 /* disable attribute target */ #endif XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { XXH_ALIGN(64) __m512i* const xacc = (__m512i*)acc; XXH_ASSERT((((size_t)acc) & 63) == 0); XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); { /* data_vec = input[0]; */ __m512i const data_vec = _mm512_loadu_si512(input); /* key_vec = secret[0]; */ __m512i const key_vec = _mm512_loadu_si512(secret); /* data_key = data_vec ^ key_vec; */ __m512i const data_key = _mm512_xor_si512(data_vec, key_vec); /* data_key_lo = data_key >> 32; */ __m512i const data_key_lo = _mm512_shuffle_epi32(data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ __m512i const product = _mm512_mul_epu32(data_key, data_key_lo); /* xacc[0] += swap(data_vec); */ __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); __m512i const sum = _mm512_add_epi64(*xacc, data_swap); /* xacc[0] += product; */ *xacc = _mm512_add_epi64(product, sum); } } /* * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. * * Multiplication isn't perfect, as explained by Google in HighwayHash: * * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to * // varying degrees. In descending order of goodness, bytes * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. * // As expected, the upper and lower bytes are much worse. * * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 * * Since our algorithm uses a pseudorandom secret to add some variance into the * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. * * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid * extraction. * * Both XXH3_64bits and XXH3_128bits use this subroutine. */ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 63) == 0); XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); { XXH_ALIGN(64) __m512i* const xacc = (__m512i*)acc; const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); /* xacc[0] ^= (xacc[0] >> 47) */ __m512i const acc_vec = *xacc; __m512i const shifted = _mm512_srli_epi64(acc_vec, 47); __m512i const data_vec = _mm512_xor_si512(acc_vec, shifted); /* xacc[0] ^= secret; */ __m512i const key_vec = _mm512_loadu_si512(secret); __m512i const data_key = _mm512_xor_si512(data_vec, key_vec); /* xacc[0] *= XXH_PRIME32_1; */ __m512i const data_key_hi = _mm512_shuffle_epi32(data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); __m512i const prod_lo = _mm512_mul_epu32(data_key, prime32); __m512i const prod_hi = _mm512_mul_epu32(data_key_hi, prime32); *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); } } XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); XXH_ASSERT(((size_t)customSecret & 63) == 0); (void)(&XXH_writeLE64); { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, -(xxh_i64)seed64); XXH_ALIGN(64) const __m512i* const src = (const __m512i*)XXH3_kSecret; XXH_ALIGN(64) __m512i* const dest = (__m512i*)customSecret; int i; for (i = 0; i < nbRounds; ++i) { /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*', * this will warn "discards ‘const’ qualifier". */ union { XXH_ALIGN(64) const __m512i* cp; XXH_ALIGN(64) void* p; } remote_const_void; remote_const_void.cp = src + i; dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); } } } #endif #if (XXH_VECTOR == XXH_AVX2) || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) #ifndef XXH_TARGET_AVX2 #define XXH_TARGET_AVX2 /* disable attribute target */ #endif XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 31) == 0); { XXH_ALIGN(32) __m256i* const xacc = (__m256i*)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ const __m256i* const xinput = (const __m256i*)input; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ const __m256i* const xsecret = (const __m256i*)secret; size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { /* data_vec = xinput[i]; */ __m256i const data_vec = _mm256_loadu_si256(xinput + i); /* key_vec = xsecret[i]; */ __m256i const key_vec = _mm256_loadu_si256(xsecret + i); /* data_key = data_vec ^ key_vec; */ __m256i const data_key = _mm256_xor_si256(data_vec, key_vec); /* data_key_lo = data_key >> 32; */ __m256i const data_key_lo = _mm256_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ __m256i const product = _mm256_mul_epu32(data_key, data_key_lo); /* xacc[i] += swap(data_vec); */ __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); /* xacc[i] += product; */ xacc[i] = _mm256_add_epi64(product, sum); } } } XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 31) == 0); { XXH_ALIGN(32) __m256i* const xacc = (__m256i*)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ const __m256i* const xsecret = (const __m256i*)secret; const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { /* xacc[i] ^= (xacc[i] >> 47) */ __m256i const acc_vec = xacc[i]; __m256i const shifted = _mm256_srli_epi64(acc_vec, 47); __m256i const data_vec = _mm256_xor_si256(acc_vec, shifted); /* xacc[i] ^= xsecret; */ __m256i const key_vec = _mm256_loadu_si256(xsecret + i); __m256i const data_key = _mm256_xor_si256(data_vec, key_vec); /* xacc[i] *= XXH_PRIME32_1; */ __m256i const data_key_hi = _mm256_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); __m256i const prod_lo = _mm256_mul_epu32(data_key, prime32); __m256i const prod_hi = _mm256_mul_epu32(data_key_hi, prime32); xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); } } } XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); (void)(&XXH_writeLE64); XXH_PREFETCH(customSecret); { __m256i const seed = _mm256_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64, -(xxh_i64)seed64, (xxh_i64)seed64); XXH_ALIGN(64) const __m256i* const src = (const __m256i*)XXH3_kSecret; XXH_ALIGN(64) __m256i* dest = (__m256i*)customSecret; #if defined(__GNUC__) || defined(__clang__) /* * On GCC & Clang, marking 'dest' as modified will cause the compiler: * - do not extract the secret from sse registers in the internal loop * - use less common registers, and avoid pushing these reg into stack * The asm hack causes Clang to assume that XXH3_kSecretPtr aliases with * customSecret, and on aarch64, this prevented LDP from merging two * loads together for free. Putting the loads together before the stores * properly generates LDP. */ __asm__("" : "+r"(dest)); #endif /* GCC -O2 need unroll loop manually */ dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src + 0), seed); dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src + 1), seed); dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src + 2), seed); dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src + 3), seed); dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src + 4), seed); dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src + 5), seed); } } #endif /* x86dispatch always generates SSE2 */ #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) #ifndef XXH_TARGET_SSE2 #define XXH_TARGET_SSE2 /* disable attribute target */ #endif XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_accumulate_512_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { /* SSE2 is just a half-scale version of the AVX2 version. */ XXH_ASSERT((((size_t)acc) & 15) == 0); { XXH_ALIGN(16) __m128i* const xacc = (__m128i*)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i* const xinput = (const __m128i*)input; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i* const xsecret = (const __m128i*)secret; size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { /* data_vec = xinput[i]; */ __m128i const data_vec = _mm_loadu_si128(xinput + i); /* key_vec = xsecret[i]; */ __m128i const key_vec = _mm_loadu_si128(xsecret + i); /* data_key = data_vec ^ key_vec; */ __m128i const data_key = _mm_xor_si128(data_vec, key_vec); /* data_key_lo = data_key >> 32; */ __m128i const data_key_lo = _mm_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ __m128i const product = _mm_mul_epu32(data_key, data_key_lo); /* xacc[i] += swap(data_vec); */ __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); __m128i const sum = _mm_add_epi64(xacc[i], data_swap); /* xacc[i] += product; */ xacc[i] = _mm_add_epi64(product, sum); } } } XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { XXH_ALIGN(16) __m128i* const xacc = (__m128i*)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i* const xsecret = (const __m128i*)secret; const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { /* xacc[i] ^= (xacc[i] >> 47) */ __m128i const acc_vec = xacc[i]; __m128i const shifted = _mm_srli_epi64(acc_vec, 47); __m128i const data_vec = _mm_xor_si128(acc_vec, shifted); /* xacc[i] ^= xsecret[i]; */ __m128i const key_vec = _mm_loadu_si128(xsecret + i); __m128i const data_key = _mm_xor_si128(data_vec, key_vec); /* xacc[i] *= XXH_PRIME32_1; */ __m128i const data_key_hi = _mm_shuffle_epi32(data_key, _MM_SHUFFLE(0, 3, 0, 1)); __m128i const prod_lo = _mm_mul_epu32(data_key, prime32); __m128i const prod_hi = _mm_mul_epu32(data_key_hi, prime32); xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); } } } XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); (void)(&XXH_writeLE64); { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); #if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 // MSVC 32bit mode does not support _mm_set_epi64x before 2015 XXH_ALIGN(16) const xxh_i64 seed64x2[2] = {(xxh_i64)seed64, -(xxh_i64)seed64}; __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); #else __m128i const seed = _mm_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64); #endif int i; XXH_ALIGN(64) const float* const src = (float const*)XXH3_kSecret; XXH_ALIGN(XXH_SEC_ALIGN) __m128i* dest = (__m128i*)customSecret; #if defined(__GNUC__) || defined(__clang__) /* * On GCC & Clang, marking 'dest' as modified will cause the compiler: * - do not extract the secret from sse registers in the internal loop * - use less common registers, and avoid pushing these reg into stack */ __asm__("" : "+r"(dest)); #endif for (i = 0; i < nbRounds; ++i) { dest[i] = _mm_add_epi64(_mm_castps_si128(_mm_load_ps(src + i * 4)), seed); } } } #endif #if (XXH_VECTOR == XXH_NEON) XXH_FORCE_INLINE void XXH3_accumulate_512_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { XXH_ALIGN(16) uint64x2_t* const xacc = (uint64x2_t*)acc; /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ uint8_t const* const xinput = (const uint8_t*)input; uint8_t const* const xsecret = (const uint8_t*)secret; size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { /* data_vec = xinput[i]; */ uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); /* key_vec = xsecret[i]; */ uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); uint64x2_t data_key; uint32x2_t data_key_lo, data_key_hi; /* xacc[i] += swap(data_vec); */ uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); uint64x2_t const swapped = vextq_u64(data64, data64, 1); xacc[i] = vaddq_u64(xacc[i], swapped); /* data_key = data_vec ^ key_vec; */ data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); * data_key_hi = (uint32x2_t) (data_key >> 32); * data_key = UNDEFINED; */ XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ xacc[i] = vmlal_u32(xacc[i], data_key_lo, data_key_hi); } } } XXH_FORCE_INLINE void XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { uint64x2_t* xacc = (uint64x2_t*)acc; uint8_t const* xsecret = (uint8_t const*)secret; uint32x2_t prime = vdup_n_u32(XXH_PRIME32_1); size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { /* xacc[i] ^= (xacc[i] >> 47); */ uint64x2_t acc_vec = xacc[i]; uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); uint64x2_t data_vec = veorq_u64(acc_vec, shifted); /* xacc[i] ^= xsecret[i]; */ uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); uint64x2_t data_key = veorq_u64(data_vec, vreinterpretq_u64_u8(key_vec)); /* xacc[i] *= XXH_PRIME32_1 */ uint32x2_t data_key_lo, data_key_hi; /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); * data_key_hi = (uint32x2_t) (xacc[i] >> 32); * xacc[i] = UNDEFINED; */ XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); { /* * prod_hi = (data_key >> 32) * XXH_PRIME32_1; * * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will * incorrectly "optimize" this: * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); * shifted = vshll_n_u32(tmp, 32); * to this: * tmp = "vmulq_u64"(a, b); // no such thing! * shifted = vshlq_n_u64(tmp, 32); * * However, unlike SSE, Clang lacks a 64-bit multiply routine * for NEON, and it scalarizes two 64-bit multiplies instead. * * vmull_u32 has the same timing as vmul_u32, and it avoids * this bug completely. * See https://bugs.llvm.org/show_bug.cgi?id=39967 */ uint64x2_t prod_hi = vmull_u32(data_key_hi, prime); /* xacc[i] = prod_hi << 32; */ xacc[i] = vshlq_n_u64(prod_hi, 32); /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); } } } } #endif #if (XXH_VECTOR == XXH_VSX) XXH_FORCE_INLINE void XXH3_accumulate_512_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { xxh_u64x2* const xacc = (xxh_u64x2*)acc; /* presumed aligned */ xxh_u64x2 const* const xinput = (xxh_u64x2 const*)input; /* no alignment restriction */ xxh_u64x2 const* const xsecret = (xxh_u64x2 const*)secret; /* no alignment restriction */ xxh_u64x2 const v32 = {32, 32}; size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { /* data_vec = xinput[i]; */ xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); /* key_vec = xsecret[i]; */ xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); xxh_u64x2 const data_key = data_vec ^ key_vec; /* shuffled = (data_key << 32) | (data_key >> 32); */ xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); xacc[i] += product; /* swap high and low halves */ #ifdef __s390x__ xacc[i] += vec_permi(data_vec, data_vec, 2); #else xacc[i] += vec_xxpermdi(data_vec, data_vec, 2); #endif } } XXH_FORCE_INLINE void XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { xxh_u64x2* const xacc = (xxh_u64x2*)acc; const xxh_u64x2* const xsecret = (const xxh_u64x2*)secret; /* constants */ xxh_u64x2 const v32 = {32, 32}; xxh_u64x2 const v47 = {47, 47}; xxh_u32x4 const prime = {XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1}; size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { /* xacc[i] ^= (xacc[i] >> 47); */ xxh_u64x2 const acc_vec = xacc[i]; xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); /* xacc[i] ^= xsecret[i]; */ xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); xxh_u64x2 const data_key = data_vec ^ key_vec; /* xacc[i] *= XXH_PRIME32_1 */ /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); xacc[i] = prod_odd + (prod_even << v32); } } } #endif /* scalar variants - universal */ XXH_FORCE_INLINE void XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*)acc; /* presumed aligned */ const xxh_u8* const xinput = (const xxh_u8*)input; /* no alignment restriction */ const xxh_u8* const xsecret = (const xxh_u8*)secret; /* no alignment restriction */ size_t i; XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN - 1)) == 0); for (i = 0; i < XXH_ACC_NB; i++) { xxh_u64 const data_val = XXH_readLE64(xinput + 8 * i); xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i * 8); xacc[i ^ 1] += data_val; /* swap adjacent lanes */ xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); } } XXH_FORCE_INLINE void XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64* const xacc = (xxh_u64*)acc; /* presumed aligned */ const xxh_u8* const xsecret = (const xxh_u8*)secret; /* no alignment restriction */ size_t i; XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN - 1)) == 0); for (i = 0; i < XXH_ACC_NB; i++) { xxh_u64 const key64 = XXH_readLE64(xsecret + 8 * i); xxh_u64 acc64 = xacc[i]; acc64 = XXH_xorshift64(acc64, 47); acc64 ^= key64; acc64 *= XXH_PRIME32_1; xacc[i] = acc64; } } XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { /* * We need a separate pointer for the hack below, * which requires a non-const pointer. * Any decent compiler will optimize this out otherwise. */ const xxh_u8* kSecretPtr = XXH3_kSecret; XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); #if defined(__clang__) && defined(__aarch64__) /* * UGLY HACK: * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are * placed sequentially, in order, at the top of the unrolled loop. * * While MOVK is great for generating constants (2 cycles for a 64-bit * constant compared to 4 cycles for LDR), long MOVK chains stall the * integer pipelines: * I L S * MOVK * MOVK * MOVK * MOVK * ADD * SUB STR * STR * By forcing loads from memory (as the asm line causes Clang to assume * that XXH3_kSecretPtr has been changed), the pipelines are used more * efficiently: * I L S * LDR * ADD LDR * SUB STR * STR * XXH3_64bits_withSeed, len == 256, Snapdragon 835 * without hack: 2654.4 MB/s * with hack: 3202.9 MB/s */ __asm__("" : "+r"(kSecretPtr)); #endif /* * Note: in debug mode, this overrides the asm optimization * and Clang will emit MOVK chains again. */ XXH_ASSERT(kSecretPtr == XXH3_kSecret); { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; int i; for (i = 0; i < nbRounds; i++) { /* * The asm hack causes Clang to assume that kSecretPtr aliases with * customSecret, and on aarch64, this prevented LDP from merging two * loads together for free. Putting the loads together before the stores * properly generates LDP. */ xxh_u64 lo = XXH_readLE64(kSecretPtr + 16 * i) + seed64; xxh_u64 hi = XXH_readLE64(kSecretPtr + 16 * i + 8) - seed64; XXH_writeLE64((xxh_u8*)customSecret + 16 * i, lo); XXH_writeLE64((xxh_u8*)customSecret + 16 * i + 8, hi); } } } typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*); typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); #if (XXH_VECTOR == XXH_AVX512) #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 #elif (XXH_VECTOR == XXH_AVX2) #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 #elif (XXH_VECTOR == XXH_SSE2) #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 #elif (XXH_VECTOR == XXH_NEON) #define XXH3_accumulate_512 XXH3_accumulate_512_neon #define XXH3_scrambleAcc XXH3_scrambleAcc_neon #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #elif (XXH_VECTOR == XXH_VSX) #define XXH3_accumulate_512 XXH3_accumulate_512_vsx #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #else /* scalar */ #define XXH3_accumulate_512 XXH3_accumulate_512_scalar #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #endif #ifndef XXH_PREFETCH_DIST #ifdef __clang__ #define XXH_PREFETCH_DIST 320 #else #if (XXH_VECTOR == XXH_AVX512) #define XXH_PREFETCH_DIST 512 #else #define XXH_PREFETCH_DIST 384 #endif #endif /* __clang__ */ #endif /* XXH_PREFETCH_DIST */ /* * XXH3_accumulate() * Loops over XXH3_accumulate_512(). * Assumption: nbStripes will not overflow the secret size */ XXH_FORCE_INLINE void XXH3_accumulate(xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT input, const xxh_u8* XXH_RESTRICT secret, size_t nbStripes, XXH3_f_accumulate_512 f_acc512) { size_t n; for (n = 0; n < nbStripes; n++) { const xxh_u8* const in = input + n * XXH_STRIPE_LEN; XXH_PREFETCH(in + XXH_PREFETCH_DIST); f_acc512(acc, in, secret + n * XXH_SECRET_CONSUME_RATE); } } XXH_FORCE_INLINE void XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; size_t const nb_blocks = (len - 1) / block_len; size_t n; XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); for (n = 0; n < nb_blocks; n++) { XXH3_accumulate(acc, input + n * block_len, secret, nbStripesPerBlock, f_acc512); f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); } /* last partial block */ XXH_ASSERT(len > XXH_STRIPE_LEN); { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); XXH3_accumulate(acc, input + nb_blocks * block_len, secret, nbStripes, f_acc512); /* last stripe */ { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; #define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); } } } XXH_FORCE_INLINE xxh_u64 XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) { return XXH3_mul128_fold64(acc[0] ^ XXH_readLE64(secret), acc[1] ^ XXH_readLE64(secret + 8)); } static XXH64_hash_t XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) { xxh_u64 result64 = start; size_t i = 0; for (i = 0; i < 4; i++) { result64 += XXH3_mix2Accs(acc + 2 * i, secret + 16 * i); #if defined(__clang__) /* Clang */ \ && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ /* * UGLY HACK: * Prevent autovectorization on Clang ARMv7-a. Exact same problem as * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. * XXH3_64bits, len == 256, Snapdragon 835: * without hack: 2063.7 MB/s * with hack: 2560.7 MB/s */ __asm__("" : "+r"(result64)); #endif } return XXH3_avalanche(result64); } #define XXH3_INIT_ACC \ {XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1} XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, const void* XXH_RESTRICT secret, size_t secretSize, XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble); /* converge into final hash */ XXH_STATIC_ASSERT(sizeof(acc) == 64); /* do not align on 8, so that the secret is different from the accumulator */ #define XXH_SECRET_MERGEACCS_START 11 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); } /* * It's important for performance that XXH3_hashLong is not inlined. */ XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) { (void)seed64; return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); } /* * It's important for performance that XXH3_hashLong is not inlined. * Since the function is not inlined, the compiler may not be able to understand that, * in some scenarios, its `secret` argument is actually a compile time constant. * This variant enforces that the compiler can detect that, * and uses this opportunity to streamline the generated code for better performance. */ XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) { (void)seed64; (void)secret; (void)secretLen; return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); } /* * XXH3_hashLong_64b_withSeed(): * Generate a custom key based on alteration of default XXH3_kSecret with the seed, * and then use this key for long mode hashing. * * This operation is decently fast but nonetheless costs a little bit of time. * Try to avoid it whenever possible (typically when seed==0). * * It's important for performance that XXH3_hashLong is not inlined. Not sure * why (uop cache maybe?), but the difference is large and easily measurable. */ XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, XXH64_hash_t seed, XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble, XXH3_f_initCustomSecret f_initSec) { if (seed == 0) return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), f_acc512, f_scramble); { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; f_initSec(secret, seed); return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), f_acc512, f_scramble); } } /* * It's important for performance that XXH3_hashLong is not inlined. */ XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed(const void* input, size_t len, XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen) { (void)secret; (void)secretLen; return XXH3_hashLong_64b_withSeed_internal(input, len, seed, XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); } typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); XXH_FORCE_INLINE XXH64_hash_t XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, XXH3_hashLong64_f f_hashLong) { XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); /* * If an action is to be taken if `secretLen` condition is not respected, * it should be done here. * For now, it's a contract pre-condition. * Adding a check and a branch here would cost performance at every hash. * Also, note that function signature doesn't offer room to return an error. */ if (len <= 16) return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); if (len <= 128) return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); } /* === Public entry point === */ /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) { return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) { return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) { return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); } /* === XXH3 streaming === */ /* * Malloc's a pointer that is always aligned to align. * * This must be freed with `XXH_alignedFree()`. * * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. * * This underalignment previously caused a rather obvious crash which went * completely unnoticed due to XXH3_createState() not actually being tested. * Credit to RedSpah for noticing this bug. * * The alignment is done manually: Functions like posix_memalign or _mm_malloc * are avoided: To maintain portability, we would have to write a fallback * like this anyways, and besides, testing for the existence of library * functions without relying on external build tools is impossible. * * The method is simple: Overallocate, manually align, and store the offset * to the original behind the returned pointer. * * Align must be a power of 2 and 8 <= align <= 128. */ static void* XXH_alignedMalloc(size_t s, size_t align) { XXH_ASSERT(align <= 128 && align >= 8); /* range check */ XXH_ASSERT((align & (align - 1)) == 0); /* power of 2 */ XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ { /* Overallocate to make room for manual realignment and an offset byte */ xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); if (base != NULL) { /* * Get the offset needed to align this pointer. * * Even if the returned pointer is aligned, there will always be * at least one byte to store the offset to the original pointer. */ size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ /* Add the offset for the now-aligned pointer */ xxh_u8* ptr = base + offset; XXH_ASSERT((size_t)ptr % align == 0); /* Store the offset immediately before the returned pointer. */ ptr[-1] = (xxh_u8)offset; return ptr; } return NULL; } } /* * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. */ static void XXH_alignedFree(void* p) { if (p != NULL) { xxh_u8* ptr = (xxh_u8*)p; /* Get the offset byte we added in XXH_malloc. */ xxh_u8 offset = ptr[-1]; /* Free the original malloc'd pointer */ xxh_u8* base = ptr - offset; XXH_free(base); } } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) { XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); if (state == NULL) return NULL; XXH3_INITSTATE(state); return state; } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) { XXH_alignedFree(statePtr); return XXH_OK; } /*! @ingroup xxh3_family */ XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) { memcpy(dst_state, src_state, sizeof(*dst_state)); } static void XXH3_reset_internal(XXH3_state_t* statePtr, XXH64_hash_t seed, const void* secret, size_t secretSize) { size_t const initStart = offsetof(XXH3_state_t, bufferedSize); size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); XXH_ASSERT(statePtr != NULL); /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ memset((char*)statePtr + initStart, 0, initLength); statePtr->acc[0] = XXH_PRIME32_3; statePtr->acc[1] = XXH_PRIME64_1; statePtr->acc[2] = XXH_PRIME64_2; statePtr->acc[3] = XXH_PRIME64_3; statePtr->acc[4] = XXH_PRIME64_4; statePtr->acc[5] = XXH_PRIME32_2; statePtr->acc[6] = XXH_PRIME64_5; statePtr->acc[7] = XXH_PRIME32_1; statePtr->seed = seed; statePtr->extSecret = (const unsigned char*)secret; XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr) { if (statePtr == NULL) return XXH_ERROR; XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) { if (statePtr == NULL) return XXH_ERROR; XXH3_reset_internal(statePtr, 0, secret, secretSize); if (secret == NULL) return XXH_ERROR; if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; return XXH_OK; } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) { if (statePtr == NULL) return XXH_ERROR; if (seed == 0) return XXH3_64bits_reset(statePtr); if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } /* Note : when XXH3_consumeStripes() is invoked, * there must be a guarantee that at least one more byte must be consumed from input * so that the function can blindly consume all stripes using the "normal" secret segment */ XXH_FORCE_INLINE void XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, const xxh_u8* XXH_RESTRICT input, size_t nbStripes, const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */ XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { /* need a scrambling operation */ size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512); f_scramble(acc, secret + secretLimit); XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512); *nbStripesSoFarPtr = nbStripesAfterBlock; } else { XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); *nbStripesSoFarPtr += nbStripes; } } /* * Both XXH3_64bits_update and XXH3_128bits_update use this routine. */ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t* state, const xxh_u8* input, size_t len, XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { if (input == NULL) #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) return XXH_OK; #else return XXH_ERROR; #endif { const xxh_u8* const bEnd = input + len; const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; state->totalLen += len; XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { /* fill in tmp buffer */ XXH_memcpy(state->buffer + state->bufferedSize, input, len); state->bufferedSize += (XXH32_hash_t)len; return XXH_OK; } /* total input is now > XXH3_INTERNALBUFFER_SIZE */ #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ /* * Internal buffer is partially filled (always, except at beginning) * Complete it, then consume it. */ if (state->bufferedSize) { size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); input += loadSize; XXH3_consumeStripes(state->acc, &state->nbStripesSoFar, state->nbStripesPerBlock, state->buffer, XXH3_INTERNALBUFFER_STRIPES, secret, state->secretLimit, f_acc512, f_scramble); state->bufferedSize = 0; } XXH_ASSERT(input < bEnd); /* Consume input by a multiple of internal buffer size */ if (input + XXH3_INTERNALBUFFER_SIZE < bEnd) { const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; do { XXH3_consumeStripes(state->acc, &state->nbStripesSoFar, state->nbStripesPerBlock, input, XXH3_INTERNALBUFFER_STRIPES, secret, state->secretLimit, f_acc512, f_scramble); input += XXH3_INTERNALBUFFER_SIZE; } while (input < limit); /* for last partial stripe */ memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); } XXH_ASSERT(input < bEnd); /* Some remaining input (always) : buffer it */ XXH_memcpy(state->buffer, input, (size_t)(bEnd - input)); state->bufferedSize = (XXH32_hash_t)(bEnd - input); } return XXH_OK; } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) { return XXH3_update(state, (const xxh_u8*)input, len, XXH3_accumulate_512, XXH3_scrambleAcc); } XXH_FORCE_INLINE void XXH3_digest_long(XXH64_hash_t* acc, const XXH3_state_t* state, const unsigned char* secret) { /* * Digest on a local copy. This way, the state remains unaltered, and it can * continue ingesting more input afterwards. */ memcpy(acc, state->acc, sizeof(state->acc)); if (state->bufferedSize >= XXH_STRIPE_LEN) { size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; size_t nbStripesSoFar = state->nbStripesSoFar; XXH3_consumeStripes(acc, &nbStripesSoFar, state->nbStripesPerBlock, state->buffer, nbStripes, secret, state->secretLimit, XXH3_accumulate_512, XXH3_scrambleAcc); /* last stripe */ XXH3_accumulate_512(acc, state->buffer + state->bufferedSize - XXH_STRIPE_LEN, secret + state->secretLimit - XXH_SECRET_LASTACC_START); } else { /* bufferedSize < XXH_STRIPE_LEN */ xxh_u8 lastStripe[XXH_STRIPE_LEN]; size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); XXH3_accumulate_512(acc, lastStripe, secret + state->secretLimit - XXH_SECRET_LASTACC_START); } } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t* state) { const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; if (state->totalLen > XXH3_MIDSIZE_MAX) { XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; XXH3_digest_long(acc, state, secret); return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)state->totalLen * XXH_PRIME64_1); } /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ if (state->seed) return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), secret, state->secretLimit + XXH_STRIPE_LEN); } #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) /*! @ingroup xxh3_family */ XXH_PUBLIC_API void XXH3_generateSecret(void* secretBuffer, const void* customSeed, size_t customSeedSize) { XXH_ASSERT(secretBuffer != NULL); if (customSeedSize == 0) { memcpy(secretBuffer, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); return; } XXH_ASSERT(customSeed != NULL); { size_t const segmentSize = sizeof(XXH128_hash_t); size_t const nbSegments = XXH_SECRET_DEFAULT_SIZE / segmentSize; XXH128_canonical_t scrambler; XXH64_hash_t seeds[12]; size_t segnb; XXH_ASSERT(nbSegments == 12); XXH_ASSERT(segmentSize * nbSegments == XXH_SECRET_DEFAULT_SIZE); /* exact multiple */ XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); /* * Copy customSeed to seeds[], truncating or repeating as necessary. */ { size_t toFill = XXH_MIN(customSeedSize, sizeof(seeds)); size_t filled = toFill; memcpy(seeds, customSeed, toFill); while (filled < sizeof(seeds)) { toFill = XXH_MIN(filled, sizeof(seeds) - filled); memcpy((char*)seeds + filled, seeds, toFill); filled += toFill; } } /* generate secret */ memcpy(secretBuffer, &scrambler, sizeof(scrambler)); for (segnb = 1; segnb < nbSegments; segnb++) { size_t const segmentStart = segnb * segmentSize; XXH128_canonical_t segment; XXH128_canonicalFromHash(&segment, XXH128(&scrambler, sizeof(scrambler), XXH_readLE64(seeds + segnb) + segnb)); memcpy((char*)secretBuffer + segmentStart, &segment, sizeof(segment)); } } } /* ========================================== * XXH3 128 bits (a.k.a XXH128) * ========================================== * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, * even without counting the significantly larger output size. * * For example, extra steps are taken to avoid the seed-dependent collisions * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). * * This strength naturally comes at the cost of some speed, especially on short * lengths. Note that longer hashes are about as fast as the 64-bit version * due to it using only a slight modification of the 64-bit loop. * * XXH128 is also more oriented towards 64-bit machines. It is still extremely * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). */ XXH_FORCE_INLINE XXH128_hash_t XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { /* A doubled version of 1to3_64b with different constants. */ XXH_ASSERT(input != NULL); XXH_ASSERT(1 <= len && len <= 3); XXH_ASSERT(secret != NULL); /* * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } */ { xxh_u8 const c1 = input[0]; xxh_u8 const c2 = input[len >> 1]; xxh_u8 const c3 = input[len - 1]; xxh_u32 const combinedl = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret + 4)) + seed; xxh_u64 const bitfliph = (XXH_readLE32(secret + 8) ^ XXH_readLE32(secret + 12)) - seed; xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; XXH128_hash_t h128; h128.low64 = XXH64_avalanche(keyed_lo); h128.high64 = XXH64_avalanche(keyed_hi); return h128; } } XXH_FORCE_INLINE XXH128_hash_t XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); XXH_ASSERT(4 <= len && len <= 8); seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; { xxh_u32 const input_lo = XXH_readLE32(input); xxh_u32 const input_hi = XXH_readLE32(input + len - 4); xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); xxh_u64 const bitflip = (XXH_readLE64(secret + 16) ^ XXH_readLE64(secret + 24)) + seed; xxh_u64 const keyed = input_64 ^ bitflip; /* Shift len to the left to ensure it is even, this avoids even multiplies. */ XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); m128.high64 += (m128.low64 << 1); m128.low64 ^= (m128.high64 >> 3); m128.low64 = XXH_xorshift64(m128.low64, 35); m128.low64 *= 0x9FB21C651E98DF25ULL; m128.low64 = XXH_xorshift64(m128.low64, 28); m128.high64 = XXH3_avalanche(m128.high64); return m128; } } XXH_FORCE_INLINE XXH128_hash_t XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); XXH_ASSERT(9 <= len && len <= 16); { xxh_u64 const bitflipl = (XXH_readLE64(secret + 32) ^ XXH_readLE64(secret + 40)) - seed; xxh_u64 const bitfliph = (XXH_readLE64(secret + 48) ^ XXH_readLE64(secret + 56)) + seed; xxh_u64 const input_lo = XXH_readLE64(input); xxh_u64 input_hi = XXH_readLE64(input + len - 8); XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); /* * Put len in the middle of m128 to ensure that the length gets mixed to * both the low and high bits in the 128x64 multiply below. */ m128.low64 += (xxh_u64)(len - 1) << 54; input_hi ^= bitfliph; /* * Add the high 32 bits of input_hi to the high 32 bits of m128, then * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to * the high 64 bits of m128. * * The best approach to this operation is different on 32-bit and 64-bit. */ if (sizeof(void*) < sizeof(xxh_u64)) { /* 32-bit */ /* * 32-bit optimized version, which is more readable. * * On 32-bit, it removes an ADC and delays a dependency between the two * halves of m128.high64, but it generates an extra mask on 64-bit. */ m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); } else { /* * 64-bit optimized (albeit more confusing) version. * * Uses some properties of addition and multiplication to remove the mask: * * Let: * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) * c = XXH_PRIME32_2 * * a + (b * c) * Inverse Property: x + y - x == y * a + (b * (1 + c - 1)) * Distributive Property: x * (y + z) == (x * y) + (x * z) * a + (b * 1) + (b * (c - 1)) * Identity Property: x * 1 == x * a + b + (b * (c - 1)) * * Substitute a, b, and c: * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) * * Since input_hi.hi + input_hi.lo == input_hi, we get this: * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) */ m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); } /* m128 ^= XXH_swap64(m128 >> 64); */ m128.low64 ^= XXH_swap64(m128.high64); { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); h128.high64 += m128.high64 * XXH_PRIME64_2; h128.low64 = XXH3_avalanche(h128.low64); h128.high64 = XXH3_avalanche(h128.high64); return h128; } } } /* * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN */ XXH_FORCE_INLINE XXH128_hash_t XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(len <= 16); { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); if (len) return XXH3_len_1to3_128b(input, len, secret, seed); { XXH128_hash_t h128; xxh_u64 const bitflipl = XXH_readLE64(secret + 64) ^ XXH_readLE64(secret + 72); xxh_u64 const bitfliph = XXH_readLE64(secret + 80) ^ XXH_readLE64(secret + 88); h128.low64 = XXH64_avalanche(seed ^ bitflipl); h128.high64 = XXH64_avalanche(seed ^ bitfliph); return h128; } } } /* * A bit slower than XXH3_mix16B, but handles multiply by zero better. */ XXH_FORCE_INLINE XXH128_hash_t XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, const xxh_u8* secret, XXH64_hash_t seed) { acc.low64 += XXH3_mix16B(input_1, secret + 0, seed); acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); acc.high64 += XXH3_mix16B(input_2, secret + 16, seed); acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); return acc; } XXH_FORCE_INLINE XXH128_hash_t XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; XXH_ASSERT(16 < len && len <= 128); { XXH128_hash_t acc; acc.low64 = len * XXH_PRIME64_1; acc.high64 = 0; if (len > 32) { if (len > 64) { if (len > 96) { acc = XXH128_mix32B(acc, input + 48, input + len - 64, secret + 96, seed); } acc = XXH128_mix32B(acc, input + 32, input + len - 48, secret + 64, seed); } acc = XXH128_mix32B(acc, input + 16, input + len - 32, secret + 32, seed); } acc = XXH128_mix32B(acc, input, input + len - 16, secret, seed); { XXH128_hash_t h128; h128.low64 = acc.low64 + acc.high64; h128.high64 = (acc.low64 * XXH_PRIME64_1) + (acc.high64 * XXH_PRIME64_4) + ((len - seed) * XXH_PRIME64_2); h128.low64 = XXH3_avalanche(h128.low64); h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); return h128; } } } XXH_NO_INLINE XXH128_hash_t XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); { XXH128_hash_t acc; int const nbRounds = (int)len / 32; int i; acc.low64 = len * XXH_PRIME64_1; acc.high64 = 0; for (i = 0; i < 4; i++) { acc = XXH128_mix32B(acc, input + (32 * i), input + (32 * i) + 16, secret + (32 * i), seed); } acc.low64 = XXH3_avalanche(acc.low64); acc.high64 = XXH3_avalanche(acc.high64); XXH_ASSERT(nbRounds >= 4); for (i = 4; i < nbRounds; i++) { acc = XXH128_mix32B(acc, input + (32 * i), input + (32 * i) + 16, secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), seed); } /* last bytes */ acc = XXH128_mix32B(acc, input + len - 16, input + len - 32, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, 0ULL - seed); { XXH128_hash_t h128; h128.low64 = acc.low64 + acc.high64; h128.high64 = (acc.low64 * XXH_PRIME64_1) + (acc.high64 * XXH_PRIME64_4) + ((len - seed) * XXH_PRIME64_2); h128.low64 = XXH3_avalanche(h128.low64); h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); return h128; } } } XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble) { XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble); /* converge into final hash */ XXH_STATIC_ASSERT(sizeof(acc) == 64); XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); { XXH128_hash_t h128; h128.low64 = XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); h128.high64 = XXH3_mergeAccs(acc, secret + secretSize - sizeof(acc) - XXH_SECRET_MERGEACCS_START, ~((xxh_u64)len * XXH_PRIME64_2)); return h128; } } /* * It's important for performance that XXH3_hashLong is not inlined. */ XXH_NO_INLINE XXH128_hash_t XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) { (void)seed64; (void)secret; (void)secretLen; return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); } /* * It's important for performance that XXH3_hashLong is not inlined. */ XXH_NO_INLINE XXH128_hash_t XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) { (void)seed64; return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); } XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble, XXH3_f_initCustomSecret f_initSec) { if (seed64 == 0) return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), f_acc512, f_scramble); { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; f_initSec(secret, seed64); return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), f_acc512, f_scramble); } } /* * It's important for performance that XXH3_hashLong is not inlined. */ XXH_NO_INLINE XXH128_hash_t XXH3_hashLong_128b_withSeed(const void* input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) { (void)secret; (void)secretLen; return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); } typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, XXH64_hash_t, const void* XXH_RESTRICT, size_t); XXH_FORCE_INLINE XXH128_hash_t XXH3_128bits_internal(const void* input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, XXH3_hashLong128_f f_hl128) { XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); /* * If an action is to be taken if `secret` conditions are not respected, * it should be done here. * For now, it's a contract pre-condition. * Adding a check and a branch here would cost performance at every hash. */ if (len <= 16) return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); if (len <= 128) return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); return f_hl128(input, len, seed64, secret, secretLen); } /* === Public XXH128 API === */ /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) { return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_default); } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) { return XXH3_128bits_internal(input, len, 0, (const xxh_u8*)secret, secretSize, XXH3_hashLong_128b_withSecret); } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) { return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed); } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH128(const void* input, size_t len, XXH64_hash_t seed) { return XXH3_128bits_withSeed(input, len, seed); } /* === XXH3 128-bit streaming === */ /* * All the functions are actually the same as for 64-bit streaming variant. * The only difference is the finalization routine. */ /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr) { if (statePtr == NULL) return XXH_ERROR; XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) { if (statePtr == NULL) return XXH_ERROR; XXH3_reset_internal(statePtr, 0, secret, secretSize); if (secret == NULL) return XXH_ERROR; if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; return XXH_OK; } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) { if (statePtr == NULL) return XXH_ERROR; if (seed == 0) return XXH3_128bits_reset(statePtr); if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) { return XXH3_update(state, (const xxh_u8*)input, len, XXH3_accumulate_512, XXH3_scrambleAcc); } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t* state) { const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; if (state->totalLen > XXH3_MIDSIZE_MAX) { XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; XXH3_digest_long(acc, state, secret); XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); { XXH128_hash_t h128; h128.low64 = XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)state->totalLen * XXH_PRIME64_1); h128.high64 = XXH3_mergeAccs( acc, secret + state->secretLimit + XXH_STRIPE_LEN - sizeof(acc) - XXH_SECRET_MERGEACCS_START, ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); return h128; } } /* len <= XXH3_MIDSIZE_MAX : short code */ if (state->seed) return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), secret, state->secretLimit + XXH_STRIPE_LEN); } /* 128-bit utility functions */ #include /* memcmp, memcpy */ /* return : 1 is equal, 0 if different */ /*! @ingroup xxh3_family */ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { /* note : XXH128_hash_t is compact, it has no padding byte */ return !(memcmp(&h1, &h2, sizeof(h1))); } /* This prototype is compatible with stdlib's qsort(). * return : >0 if *h128_1 > *h128_2 * <0 if *h128_1 < *h128_2 * =0 if *h128_1 == *h128_2 */ /*! @ingroup xxh3_family */ XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) { XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); /* note : bets that, in most cases, hash values are different */ if (hcmp) return hcmp; return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); } /*====== Canonical representation ======*/ /*! @ingroup xxh3_family */ XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) { hash.high64 = XXH_swap64(hash.high64); hash.low64 = XXH_swap64(hash.low64); } memcpy(dst, &hash.high64, sizeof(hash.high64)); memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); } /*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src) { XXH128_hash_t h; h.high64 = XXH_readBE64(src); h.low64 = XXH_readBE64(src->digest + 8); return h; } /* Pop our optimization override from above */ #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ #pragma GCC pop_options #endif #endif /* XXH_NO_LONG_LONG */ /*! * @} */ #endif /* XXH_IMPLEMENTATION */ #if defined(__cplusplus) } #endif eckit-2.0.7/src/eckit/contrib/xxhash/LICENSE0000664000175000017500000000255515161702250020647 0ustar alastairalastairxxHash Library Copyright (c) 2012-2020 Yann Collet All rights reserved. BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. eckit-2.0.7/src/eckit/spec/0000775000175000017500000000000015161702250015622 5ustar alastairalastaireckit-2.0.7/src/eckit/spec/LibEcKitSpec.cc0000664000175000017500000000200015161702250020362 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/spec/LibEcKitSpec.h" #include "eckit/eckit_version.h" namespace eckit { REGISTER_LIBRARY(LibEcKitSpec); LibEcKitSpec::LibEcKitSpec() : Library("eckit_spec") {} LibEcKitSpec& LibEcKitSpec::instance() { static LibEcKitSpec lib; return lib; } const void* LibEcKitSpec::addr() const { return this; } std::string LibEcKitSpec::version() const { return eckit_version_str(); } std::string LibEcKitSpec::gitsha1(unsigned int count) const { std::string sha1(eckit_git_sha1()); return sha1.empty() ? "not available" : sha1.substr(0, std::min(count, 40U)); } } // namespace eckit eckit-2.0.7/src/eckit/spec/Spec.cc0000664000175000017500000000663015161702250017030 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/spec/Spec.h" #include #include "eckit/log/JSON.h" #include "eckit/spec/Exceptions.h" namespace eckit::spec { template static inline T _get_d(const Spec& spec, const std::string& name, const T& _default) { T value{_default}; spec.has(name) && spec.get(name, value); return value; } template static inline T _get_t(const Spec& spec, const std::string& name) { T value{}; return spec.get(name, value) ? value : throw exception::SpecError(name, Here()); } std::string Spec::get_string(const std::string& name) const { return _get_t(*this, name); } bool Spec::get_bool(const std::string& name) const { return _get_t(*this, name); } int Spec::get_int(const std::string& name) const { return _get_t(*this, name); } long Spec::get_long(const std::string& name) const { return _get_t(*this, name); } size_t Spec::get_unsigned(const std::string& name) const { return _get_t(*this, name); } double Spec::get_double(const std::string& name) const { return _get_t(*this, name); } std::vector Spec::get_long_vector(const std::string& name) const { return _get_t>(*this, name); } std::vector Spec::get_unsigned_vector(const std::string& name) const { return _get_t>(*this, name); } std::vector Spec::get_double_vector(const std::string& name) const { return _get_t>(*this, name); } std::string Spec::get_string(const std::string& name, const std::string& _default) const { return _get_d(*this, name, _default); } bool Spec::get_bool(const std::string& name, const bool& _default) const { return _get_d(*this, name, _default); } int Spec::get_int(const std::string& name, const int& _default) const { return _get_d(*this, name, _default); } long Spec::get_long(const std::string& name, const long& _default) const { return _get_d(*this, name, _default); } size_t Spec::get_unsigned(const std::string& name, const size_t& _default) const { return _get_d(*this, name, _default); } double Spec::get_double(const std::string& name, const double& _default) const { return _get_d(*this, name, _default); } std::vector Spec::get_long_vector(const std::string& name, const std::vector& _default) const { return _get_d(*this, name, _default); } std::vector Spec::get_unsigned_vector(const std::string& name, const std::vector& _default) const { return _get_d(*this, name, _default); } std::vector Spec::get_double_vector(const std::string& name, const std::vector& _default) const { return _get_d(*this, name, _default); } std::string Spec::str() const { std::ostringstream str; JSON j(str); json(j); return str.str(); } const Spec& Spec::spec(const std::string& name) const { throw exception::SpecError(name, Here()); } void Spec::print(std::ostream& out) const { JSON j(out); json(j); } } // namespace eckit::spec eckit-2.0.7/src/eckit/spec/Custom.cc0000664000175000017500000003650315161702250017412 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/spec/Custom.h" #include #include #include #include "eckit/log/JSON.h" #include "eckit/spec/Exceptions.h" #include "eckit/value/Content.h" // for ValueList, ValueMap #include "eckit/value/Value.h" namespace eckit::spec { namespace { constexpr int STREAM_PRECISION = 15; template bool get_t_s(const From& from, To& to) { to = static_cast(from); return true; } template bool get_t_s(const From& from, std::string& to) { to = std::to_string(from); return true; } template bool get_t_s(const From& from, From& to) { to = from; return true; } template bool get_t_v(const std::vector& from, std::vector& to) { to.clear(); for (const auto& f : from) { to.emplace_back(static_cast(f)); } return true; } template bool get_t_v(const std::vector& from, std::vector& to) { to.clear(); for (const auto& f : from) { to.emplace_back(std::to_string(f)); } return true; } template bool get_t_s_integral(const Custom::container_type& map, const std::string& name, To& value) { if (auto it = map.find(name); it != map.cend()) { const auto& v = it->second; return std::holds_alternative(v) ? get_t_s(std::get(v), value) : std::holds_alternative(v) ? get_t_s(std::get(v), value) : std::holds_alternative(v) ? get_t_s(std::get(v), value) : std::holds_alternative(v) ? get_t_s(std::get(v), value) : false; } return false; } template bool get_t_s_real(const Custom::container_type& map, const std::string& name, To& value) { if (get_t_s_integral(map, name, value)) { return true; } if (auto it = map.find(name); it != map.cend()) { const auto& v = it->second; return std::holds_alternative(v) ? get_t_s(std::get(v), value) : std::holds_alternative(v) ? get_t_s(std::get(v), value) : false; } return false; } template bool get_t_v_integral(const Custom::container_type& map, const std::string& name, To& value) { if (auto it = map.find(name); it != map.cend()) { const auto& v = it->second; return std::holds_alternative>(v) ? get_t_v(std::get>(v), value) : std::holds_alternative>(v) ? get_t_v(std::get>(v), value) : std::holds_alternative>(v) ? get_t_v(std::get>(v), value) : std::holds_alternative>(v) ? get_t_v(std::get>(v), value) : false; } return false; } template bool get_t_v_real(const Custom::container_type& map, const std::string& name, To& value) { if (get_t_v_integral(map, name, value)) { return true; } if (auto it = map.find(name); it != map.cend()) { const auto& v = it->second; return std::holds_alternative>(v) ? get_t_v(std::get>(v), value) : std::holds_alternative>(v) ? get_t_v(std::get>(v), value) : false; } return false; } template Custom::value_type from_value_t(const Value& value) { T to; fromValue(to, value); return {to}; } void sanitise(Custom::container_type& container) { std::for_each(container.begin(), container.end(), [](auto& p) { if (auto& value = p.second; std::holds_alternative(value)) { value = std::string{std::get(value)}; } else if (std::holds_alternative(value)) { ASSERT(std::get(value)); } }); } } // namespace Custom::key_type::key_type(const std::string& s) : std::string{s} { std::transform(begin(), end(), begin(), [](unsigned char c) -> unsigned char { return std::tolower(c); }); } Custom::Custom(std::initializer_list init) : map_(init) { sanitise(map_); } Custom::Custom(const Custom::container_type& map) : map_(map) { sanitise(map_); } Custom::Custom(Custom::container_type&& map) : map_(map) { sanitise(map_); } Custom* Custom::make_from_value(const Value& value) { if (value.isString()) { return new Custom{{"grid", value.as()}}; } ASSERT(value.isMap()); auto scalar = [](const Value& value) -> value_type { return value.isNumber() ? value_type(static_cast(value)) : value.isDouble() ? value_type(static_cast(value)) : value.isString() ? static_cast(value) : throw BadValue(value, Here()); }; auto vector = [](const Value& value) -> value_type { const auto list(value.as()); ASSERT(!list.empty()); if (std::all_of(list.begin(), list.end(), [](const auto& v) { return v.isNumber(); })) { return std::vector(list.begin(), list.end()); } if (std::all_of(list.begin(), list.end(), [](const auto& v) { return v.isDouble() || v.isNumber(); })) { return std::vector(list.begin(), list.end()); } if (std::all_of(list.begin(), list.end(), [](const auto& v) { return v.isString(); })) { return std::vector(list.begin(), list.end()); } throw BadValue(value, Here()); }; auto* custom = new Custom; ASSERT(custom != nullptr); for (const auto& [key, value] : static_cast(value)) { const std::string name = key; custom->map_[name] = value.isMap() ? custom_ptr(Custom::make_from_value(value)) : value.isList() ? vector(value) : scalar(value); } return custom; } bool Custom::operator==(const Custom& other) const { auto custom_value_equal = [](const Custom& ca, const Custom& cb, const Custom::key_type& name, const auto& type_instance) -> bool { auto a = type_instance; auto b = type_instance; return ca.get(name, a) && cb.get(name, b) && a == b; }; // check every local key exists in other and is convertible to an equal value return std::all_of(map_.begin(), map_.end(), [&](const auto& _a) { const auto& name = _a.first; auto _b = other.map_.find(name); return _b != other.map_.end() && (custom_value_equal(*this, other, name, long{}) || custom_value_equal(*this, other, name, std::vector{}) || custom_value_equal(*this, other, name, double{}) || custom_value_equal(*this, other, name, std::vector{}) || custom_value_equal(*this, other, name, std::string{}) || custom_value_equal(*this, other, name, std::vector{})); }); } void Custom::set(const std::string& name, const std::string& value) { map_[name] = value; } void Custom::set(const std::string& name, bool value) { map_[name] = value; } void Custom::set(const std::string& name, int value) { map_[name] = value; } void Custom::set(const std::string& name, long value) { map_[name] = value; } void Custom::set(const std::string& name, long long value) { map_[name] = value; } void Custom::set(const std::string& name, size_t value) { map_[name] = value; } void Custom::set(const std::string& name, float value) { map_[name] = value; } void Custom::set(const std::string& name, double value) { map_[name] = value; } void Custom::set(const std::string& name, const std::vector& value) { map_[name] = value; } void Custom::set(const std::string& name, const std::vector& value) { map_[name] = value; } void Custom::set(const std::string& name, const std::vector& value) { map_[name] = value; } void Custom::set(const std::string& name, const std::vector& value) { map_[name] = value; } void Custom::set(const std::string& name, const std::vector& value) { map_[name] = value; } void Custom::set(const std::string& name, const std::vector& value) { map_[name] = value; } void Custom::set(const std::string& name, const std::vector& value) { map_[name] = value; } void Custom::set(const std::string& name, const Value& value) { using number_type = long; auto list_of = [](const ValueList& list, auto pred) { return std::all_of(list.begin(), list.end(), pred); }; auto val = value.isList() && list_of(value, [](const Value& v) { return v.isDouble(); }) ? from_value_t>(value) : value.isList() && list_of(value, [](const Value& v) { return v.isNumber(); }) ? from_value_t>(value) : value.isList() ? from_value_t>(value) : value.isDouble() ? from_value_t(value) : value.isNumber() ? from_value_t(value) : from_value_t(value); std::visit([&](const auto& val) { set(name, val); }, val); } void Custom::set(const std::string& name, Custom* value) { ASSERT(value != nullptr); map_[name] = custom_ptr(value); } void Custom::set(const std::string& name, const Spec& value) { try { const auto& custom = dynamic_cast(value); return set(name, new Custom{custom.map_}); } catch (const std::bad_cast& e) { throw exception::SpecError("Custom::set(const Spec&): Spec must be spec::Custom", Here()); } } bool Custom::has_custom(const std::string& name) const { auto it = map_.find(name); return it != map_.cend() && std::holds_alternative(it->second); } const Custom::custom_ptr& Custom::custom(const std::string& name) const { if (auto it = map_.find(name); it != map_.cend()) { if (std::holds_alternative(it->second)) { const auto& value = std::get(it->second); ASSERT(value); return value; } } throw exception::SpecError("Custom::get(" + name + ") -> custom_type& ", Here()); } void Custom::set(const std::string& name, const custom_ptr& value) { ASSERT(value); map_[name] = value; } const Spec& Custom::spec(const std::string& name) const { if (auto it = map_.find(name); it != map_.cend()) { ASSERT(std::holds_alternative(it->second)); const auto& value = std::get(it->second); ASSERT(value); return *value; } throw exception::SpecError("Custom::spec('" + name + "'): not found", Here()); } bool Custom::has(const std::string& name) const { return map_.find(name) != map_.cend(); } bool Custom::get(const std::string& name, std::string& value) const { if (auto it = map_.find(name); it != map_.cend() /*&& )*/) { if (std::holds_alternative(it->second)) { value = std::get(it->second); return true; } return get_t_s_real(map_, name, value); } return false; } bool Custom::get(const std::string& name, bool& value) const { if (auto it = map_.find(name); it != map_.cend()) { if (std::holds_alternative(it->second)) { value = std::get(it->second); return true; } if (int i = 0; get_t_s_integral(map_, name, i)) { value = i != 0; return true; } } return false; } bool Custom::get(const std::string& name, int& value) const { return get_t_s_integral(map_, name, value); } bool Custom::get(const std::string& name, long& value) const { return get_t_s_integral(map_, name, value); } bool Custom::get(const std::string& name, long long& value) const { return get_t_s_integral(map_, name, value); } bool Custom::get(const std::string& name, size_t& value) const { return get_t_s_integral(map_, name, value); } bool Custom::get(const std::string& name, float& value) const { return get_t_s_real(map_, name, value); } bool Custom::get(const std::string& name, double& value) const { return get_t_s_real(map_, name, value); } bool Custom::get(const std::string& name, std::vector& value) const { return get_t_v_integral(map_, name, value); } bool Custom::get(const std::string& name, std::vector& value) const { return get_t_v_integral(map_, name, value); } bool Custom::get(const std::string& name, std::vector& value) const { return get_t_v_integral(map_, name, value); } bool Custom::get(const std::string& name, std::vector& value) const { return get_t_v_integral(map_, name, value); } bool Custom::get(const std::string& name, std::vector& value) const { return get_t_v_real(map_, name, value); } bool Custom::get(const std::string& name, std::vector& value) const { return get_t_v_real(map_, name, value); } bool Custom::get(const std::string& name, std::vector& value) const { if (auto it = map_.find(name); it != map_.cend()) { if (std::holds_alternative>(it->second)) { value = std::get>(it->second); return true; } } return false; } void Custom::json(JSON& j) const { j.startObject(); j.precision(STREAM_PRECISION); for (const auto& [key, value] : map_) { j << key; std::visit([&](const auto& arg) { j << arg; }, value); } j.endObject(); } JSON& operator<<(JSON& j, const Custom::custom_ptr& value) { ASSERT(value); j.startObject(); for (const auto& [key, value] : value->container()) { j << key; std::visit([&](const auto& arg) { j << arg; }, value); } j.endObject(); return j; } template struct is_vector : std::false_type {}; template struct is_vector> : std::true_type {}; template constexpr bool is_vector_v = is_vector::value; std::string to_string(const Custom::value_type& value) { return std::visit( [&](const auto& arg) { std::ostringstream str; str.precision(STREAM_PRECISION); str << arg; return str.str(); }, value); } } // namespace eckit::spec eckit-2.0.7/src/eckit/spec/LibEcKitSpec.h0000664000175000017500000000152415161702250020236 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/system/Library.h" namespace eckit { class LibEcKitSpec final : public system::Library { public: // -- Methods static LibEcKitSpec& instance(); private: // -- Constructors LibEcKitSpec(); // -- Overridden methods [[nodiscard]] const void* addr() const override; std::string version() const override; std::string gitsha1(unsigned int count) const override; }; } // namespace eckit eckit-2.0.7/src/eckit/spec/Custom.h0000664000175000017500000001167615161702250017260 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include "eckit/spec/Spec.h" namespace eckit { class Value; } namespace eckit::spec { class Custom final : public Spec { public: // -- Types struct custom_ptr : std::shared_ptr { using shared_ptr::shared_ptr; }; struct key_type : std::string { key_type(const std::string&); key_type(const char* s) : key_type(std::string{s}) {} }; using value_type = std::variant, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, custom_ptr, const char* /* converted to std::string */>; using container_type = std::map; // -- Constructors Custom() = default; Custom(std::initializer_list); explicit Custom(const container_type&); explicit Custom(container_type&&); // -- Operators bool operator==(const Custom&) const; bool operator!=(const Custom& other) const { return !operator==(other); } // -- Methods const container_type& container() const { return map_; } void set(const Custom& from) { map_.insert(from.map_.begin(), from.map_.end()); } void set(const container_type& container) { map_ = container; } void set(container_type&& container) { map_.swap(container); } size_t erase(const std::string& name) { return map_.erase(name); } bool empty() const { return map_.empty(); } void clear() { map_.clear(); } bool has_custom(const std::string& name) const; const custom_ptr& custom(const std::string& name) const; void set(const std::string& name, const std::string&); void set(const std::string& name, bool); void set(const std::string& name, int); void set(const std::string& name, long); void set(const std::string& name, long long); void set(const std::string& name, size_t); void set(const std::string& name, float); void set(const std::string& name, double); void set(const std::string& name, const std::vector&); void set(const std::string& name, const std::vector&); void set(const std::string& name, const std::vector&); void set(const std::string& name, const std::vector&); void set(const std::string& name, const std::vector&); void set(const std::string& name, const std::vector&); void set(const std::string& name, const std::vector&); void set(const std::string& name, const char* value) { set(name, std::string{value}); } void set(const std::string& name, const Value&); void set(const std::string& name, Custom*); void set(const std::string& name, const Spec&); // -- Overridden methods bool has(const std::string& name) const override; bool get(const std::string& name, std::string&) const override; bool get(const std::string& name, bool&) const override; bool get(const std::string& name, int&) const override; bool get(const std::string& name, long&) const override; bool get(const std::string& name, long long&) const override; bool get(const std::string& name, size_t&) const override; bool get(const std::string& name, float&) const override; bool get(const std::string& name, double&) const override; bool get(const std::string& name, std::vector&) const override; bool get(const std::string& name, std::vector&) const override; bool get(const std::string& name, std::vector&) const override; bool get(const std::string& name, std::vector&) const override; bool get(const std::string& name, std::vector&) const override; bool get(const std::string& name, std::vector&) const override; bool get(const std::string& name, std::vector&) const override; bool only(const std::string& name) const override { return map_.size() == 1 && has(name); } void json(JSON& name) const override; const Spec& spec(const std::string& name) const override; // -- Class methods [[nodiscard]] static Custom* make_from_value(const Value&); private: // -- Members container_type map_; // -- Methods void set(const std::string& name, const custom_ptr&); }; JSON& operator<<(JSON&, const Custom::custom_ptr&); std::string to_string(const Custom::value_type&); } // namespace eckit::spec eckit-2.0.7/src/eckit/spec/Generator.h0000664000175000017500000002061315161702250017723 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include #include #include #include #include "eckit/spec/Exceptions.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" namespace eckit::spec { class Custom; class Spec; } // namespace eckit::spec namespace eckit::spec { //------------------------------------------------------------------------------------------------------ template class GeneratorT { public: // -- Types using generator_t = C; using key_t = std::string; using storage_t = std::map>; // -- Constructors GeneratorT(const GeneratorT&) = delete; GeneratorT(GeneratorT&&) = delete; // -- Operators void operator=(const GeneratorT&) = delete; void operator=(GeneratorT&&) = delete; // -- Methods static GeneratorT& instance(); bool exists(const key_t&) const; bool matches(const std::string&) const; void regist(const key_t&, generator_t*); void unregist(const key_t&); const generator_t& get(const key_t&) const; const generator_t& match(const std::string&) const; bool match(const Custom& spec, std::string& name) const { auto end = store_.cend(); auto i = end; for (auto j = store_.cbegin(); j != end; ++j) { if (!(j->first.empty()) && j->second->match(spec)) { if (i != end) { throw SeriousBug("Generator matches names '" + i->first + "' and '" + j->first + "'", Here()); } i = j; } } if (i != end) { name = i->first; ASSERT(!name.empty()); return true; } return false; } private: // -- Constructors GeneratorT() = default; // -- Members mutable Mutex mutex_; storage_t store_; // -- Methods void print(std::ostream&) const; // -- Friends friend std::ostream& operator<<(std::ostream& os, const GeneratorT& o) { o.print(os); return os; } }; //------------------------------------------------------------------------------------------------------ struct lock_type { explicit lock_type(Mutex& mutex) : lock_guard_{mutex} {} AutoLock lock_guard_; }; //------------------------------------------------------------------------------------------------------ template GeneratorT& GeneratorT::instance() { static GeneratorT obj; return obj; } template bool GeneratorT::exists(const key_t& k) const { lock_type lock(mutex_); return store_.find(k) != store_.end(); } template bool GeneratorT::matches(const std::string& k) const { lock_type lock(mutex_); return std::any_of(store_.begin(), store_.end(), [&](const auto& p) -> bool { const std::regex rex(p.first, std::regex_constants::icase); return std::regex_match(k, rex); }); } template void GeneratorT::regist(const key_t& k, generator_t* c) { lock_type lock(mutex_); if (exists(k)) { throw BadParameter("Generator has already a builder for " + k, Here()); } ASSERT(c != nullptr); store_[k].reset(c); } template void GeneratorT::unregist(const key_t& k) { lock_type lock(mutex_); if (auto it = store_.find(k); it != store_.end()) { store_.erase(it); return; } throw BadParameter("Generator unknown: '" + k + "'", Here()); } template const typename GeneratorT::generator_t& GeneratorT::get(const key_t& k) const { lock_type lock(mutex_); if (auto it = store_.find(k); it != store_.end()) { return *(it->second); } throw BadParameter("Generator unknown: '" + k + "'", Here()); } template const typename GeneratorT::generator_t& GeneratorT::match(const std::string& k) const { lock_type lock(mutex_); auto end = store_.cend(); auto i = end; for (auto j = store_.cbegin(); j != end; ++j) { if (const std::regex rex(j->first, std::regex_constants::icase); std::regex_match(k, rex)) { if (i != end) { throw SeriousBug("Generator name '" + k + "' matches '" + i->first + "' and '" + j->first + "'", Here()); } i = j; } } if (i != end) { return *(i->second); } throw BadParameter("Generator unknown: '" + k + "'", Here()); } template void GeneratorT::print(std::ostream& os) const { lock_type lock(mutex_); os << "Generator" << std::endl; int key_width = 0; for (const auto& i : store_) { key_width = std::max(static_cast(i.first.size()), key_width); } for (const auto& i : store_) { os << " " << std::setw(key_width) << std::left << i.first << " -- " << i.second.get() << std::endl; } } //------------------------------------------------------------------------------------------------------ class SpecGenerator { public: // -- Types using key_t = std::string; static constexpr const char* uid_pattern = "[0-9a-fA-F]{32}"; // -- Constructors SpecGenerator() = default; SpecGenerator(const SpecGenerator&) = delete; SpecGenerator(SpecGenerator&&) = delete; // -- Destructor virtual ~SpecGenerator() = default; // -- Operators void operator=(const SpecGenerator&) = delete; void operator=(SpecGenerator&&) = delete; // -- Methods virtual bool match(const spec::Custom&) const { return false; } }; //------------------------------------------------------------------------------------------------------ struct SpecGeneratorT0 : public SpecGenerator { [[nodiscard]] virtual Spec* spec() const = 0; }; //------------------------------------------------------------------------------------------------------ template struct SpecGeneratorT1 : public SpecGenerator { using arg1_t = ARG1; [[nodiscard]] virtual Spec* spec(arg1_t) const = 0; }; //------------------------------------------------------------------------------------------------------ template struct SpecGeneratorT2 : public SpecGenerator { using arg1_t = ARG1; using arg2_t = ARG2; [[nodiscard]] virtual Spec* spec(arg1_t, arg2_t) const = 0; }; //------------------------------------------------------------------------------------------------------ template struct ConcreteSpecGeneratorT0 final : public SpecGeneratorT0 { explicit ConcreteSpecGeneratorT0(const typename SpecGeneratorT0::key_t& key) { GeneratorT::instance().regist(key, this); } [[nodiscard]] Spec* spec() const override { return T::spec(); } }; //------------------------------------------------------------------------------------------------------ template struct ConcreteSpecGeneratorT1 final : public SpecGeneratorT1 { explicit ConcreteSpecGeneratorT1(const typename SpecGeneratorT1::key_t& key) { GeneratorT>::instance().regist(key, this); } [[nodiscard]] Spec* spec(typename SpecGeneratorT1::arg1_t p1) const override { return T::spec(p1); } }; //------------------------------------------------------------------------------------------------------ template struct ConcreteSpecGeneratorT2 final : public SpecGeneratorT2 { explicit ConcreteSpecGeneratorT2(const typename SpecGeneratorT2::key_t& key) { GeneratorT>::instance().regist(key, this); } [[nodiscard]] Spec* spec(typename SpecGeneratorT1::arg1_t p1, typename SpecGeneratorT1::arg2_t p2) const override { return T::spec(p1, p2); } }; //------------------------------------------------------------------------------------------------------ } // namespace eckit::spec eckit-2.0.7/src/eckit/spec/Spec.h0000664000175000017500000000507615161702250016675 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/config/Parametrisation.h" namespace eckit { class JSON; } namespace eckit::spec { class Spec : public Parametrisation { public: Spec() = default; ~Spec() override = default; Spec(const Spec&) = delete; Spec(Spec&&) = delete; Spec& operator=(const Spec&) = delete; Spec& operator=(Spec&&) = delete; bool operator==(const Spec& other) const { return str() == other.str(); } bool operator!=(const Spec& other) const { return !operator==(other); } std::string get_string(const std::string& name) const; bool get_bool(const std::string& name) const; int get_int(const std::string& name) const; long get_long(const std::string& name) const; size_t get_unsigned(const std::string& name) const; double get_double(const std::string& name) const; std::vector get_long_vector(const std::string& name) const; std::vector get_unsigned_vector(const std::string& name) const; std::vector get_double_vector(const std::string& name) const; std::string get_string(const std::string& name, const std::string&) const; bool get_bool(const std::string& name, const bool&) const; int get_int(const std::string& name, const int&) const; long get_long(const std::string& name, const long&) const; size_t get_unsigned(const std::string& name, const size_t&) const; double get_double(const std::string& name, const double&) const; std::vector get_long_vector(const std::string& name, const std::vector&) const; std::vector get_unsigned_vector(const std::string& name, const std::vector&) const; std::vector get_double_vector(const std::string& name, const std::vector&) const; std::string str() const; virtual bool only(const std::string& name) const { return false; } virtual void json(JSON& name) const = 0; virtual const Spec& spec(const std::string& name) const; private: virtual void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& out, const Spec& spec) { spec.print(out); return out; } }; } // namespace eckit::spec eckit-2.0.7/src/eckit/spec/Layered.h0000664000175000017500000000752115161702250017365 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/spec/Spec.h" namespace eckit::spec { class Layered final : public Spec { public: // -- Constructors Layered(); explicit Layered(const Spec&); // -- Methods void hide(const std::string&); void unhide(const std::string&); void push_back(Spec*); void push_front(Spec*); // -- Overridden methods bool has(const std::string& name) const override { return !hide_.contains(name) && (std::any_of(front_.begin(), front_.end(), [&](const decltype(front_)::value_type& c) { return c->has(name); }) || spec_.has(name) || std::any_of(back_.begin(), back_.end(), [&](const decltype(back_)::value_type& c) { return c->has(name); })); } bool get(const std::string& name, std::string& value) const override { return get_t(name, value); } bool get(const std::string& name, bool& value) const override { return get_t(name, value); } bool get(const std::string& name, int& value) const override { return get_t(name, value); } bool get(const std::string& name, long& value) const override { return get_t(name, value); } bool get(const std::string& name, long long& value) const override { return get_t(name, value); } bool get(const std::string& name, size_t& value) const override { return get_t(name, value); } bool get(const std::string& name, float& value) const override { return get_t(name, value); } bool get(const std::string& name, double& value) const override { return get_t(name, value); } bool get(const std::string& name, std::vector& value) const override { return get_t(name, value); } bool get(const std::string& name, std::vector& value) const override { return get_t(name, value); } bool get(const std::string& name, std::vector& value) const override { return get_t(name, value); } bool get(const std::string& name, std::vector& value) const override { return get_t(name, value); } bool get(const std::string& name, std::vector& value) const override { return get_t(name, value); } bool get(const std::string& name, std::vector& value) const override { return get_t(name, value); } bool get(const std::string& name, std::vector& value) const override { return get_t(name, value); } private: // -- Members struct : std::unordered_set { bool contains(const value_type& name) const { return find(name) != end(); } } hide_; const Spec& spec_; std::vector> front_; std::vector> back_; // -- Methods template bool get_t(const std::string& name, T& value) const { return !hide_.contains(name) && (std::any_of(front_.rbegin(), front_.rend(), [&](const decltype(front_)::value_type& c) { return c->get(name, value); }) || spec_.get(name, value) || std::any_of(back_.begin(), back_.end(), [&](const decltype(back_)::value_type& c) { return c->get(name, value); })); } // -- Overridden methods void print(std::ostream&) const override; bool only(const std::string& name) const override; void json(JSON&) const override; const Spec& spec(const std::string& name) const override; }; } // namespace eckit::spec eckit-2.0.7/src/eckit/spec/Exceptions.h0000664000175000017500000000120515161702250020112 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/exception/Exceptions.h" namespace eckit::spec::exception { class SpecError : public Exception { public: explicit SpecError(const std::string&, const CodeLocation&); }; } // namespace eckit::spec::exception eckit-2.0.7/src/eckit/spec/Exceptions.cc0000664000175000017500000000120615161702250020251 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/spec/Exceptions.h" namespace eckit::spec::exception { SpecError::SpecError(const std::string& what, const CodeLocation& location) : Exception("SpecError: [" + what + "]", location) {} } // namespace eckit::spec::exception eckit-2.0.7/src/eckit/spec/CMakeLists.txt0000664000175000017500000000100615161702250020357 0ustar alastairalastair ecbuild_add_library( TARGET eckit_spec TYPE SHARED INSTALL_HEADERS ALL HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/spec PUBLIC_LIBS eckit PUBLIC_INCLUDES $ $ SOURCES Custom.cc Custom.h Exceptions.cc Exceptions.h Generator.h Layered.cc Layered.h LibEcKitSpec.cc LibEcKitSpec.h Spec.cc Spec.h ) eckit-2.0.7/src/eckit/spec/Layered.cc0000664000175000017500000000464115161702250017523 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/spec/Layered.h" #include #include "eckit/log/JSON.h" #include "eckit/spec/Custom.h" #include "eckit/spec/Exceptions.h" #include "eckit/value/Value.h" namespace eckit::spec { static const Custom EMPTY; Layered::Layered() : Layered(EMPTY) {} Layered::Layered(const Spec& spec) : spec_(spec) {} void Layered::hide(const std::string& name) { hide_.insert(name); } void Layered::unhide(const std::string& name) { hide_.erase(name); } void Layered::push_back(Spec* spec) { ASSERT(spec != nullptr); back_.emplace_back(spec); } void Layered::push_front(Spec* spec) { ASSERT(spec != nullptr); front_.emplace_back(spec); } void Layered::print(std::ostream& out) const { JSON j(out); j.startObject(); j << "hide"; j.startList(); for (const auto& name : hide_) { j << name; } j.endList(); j << "before"; j.startList(); for (const auto& spec : front_) { spec->json(j); } j.endList(); j << "spec"; spec_.json(j); j << "after"; j.startList(); for (const auto& spec : back_) { spec->json(j); } j.endList(); j.endObject(); } bool Layered::only(const std::string& name) const { auto count_only = [&](const auto& c) { return c->only(name); }; return 1 == (spec_.only(name) ? 1 : 0) + // std::count_if(front_.begin(), front_.end(), count_only) + // std::count_if(back_.begin(), back_.end(), count_only); } void Layered::json(JSON&) const { NOTIMP; } const Spec& Layered::spec(const std::string& name) const { for (const auto& spec : front_) { if (spec->has(name)) { return spec->spec(name); } } if (has(name)) { return Spec::spec(name); } for (const auto& spec : back_) { if (spec->has(name)) { return spec->spec(name); } } throw exception::SpecError("Layered::spec(" + name + ")", Here()); } } // namespace eckit::spec eckit-2.0.7/src/eckit/io/0000775000175000017500000000000015161702250015277 5ustar alastairalastaireckit-2.0.7/src/eckit/io/FileBase.cc0000664000175000017500000000446415161702250017270 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/exception/Exceptions.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- template FileBase::FileBase(const PathName& path) : fd_(-1), path_(path), pos_(0) { SYSCALL2(fd_ = ::open(path.localPath(), O_RDWR | O_CREAT, 0777), path.localPath()); } template FileBase::~FileBase() { if (fd_ >= 0) SYSCALL(::close(fd_)); } template bool FileBase::read(long rec, T& data) { off_t pos = rec * sizeof(Record); if (pos != pos_) SYSCALL(pos_ = ::lseek(fd_, pos, SEEK_SET)); long size = 0; // struct flock lock = { F_RDLCK, SEEK_SET, pos_, sizeof(Record), }; // struct flock unlock = { F_UNLCK, SEEK_SET, pos_, sizeof(Record), }; // SYSCALL(::fcntl(fd_,F_SETLK,&lock)); SYSCALL(size = ::read(fd_, &buffer_, sizeof(Record))); // SYSCALL(::fcntl(fd_,F_SETLK,&unlock)); pos_ += size; if (size != sizeof(Record)) // EOF reached return false; ::memcpy(&data, &buffer_, sizeof(T)); return buffer_.valid_; } template void FileBase::write(long rec, const T& data) { off_t pos = rec * sizeof(Record); if (pos != pos_) SYSCALL(pos_ = ::lseek(fd_, pos, SEEK_SET)); long size = 0; ::memcpy(&buffer_, &data, sizeof(T)); buffer_.valid_ = true; // struct flock lock = { F_WRLCK, SEEK_SET, pos_, sizeof(Record), }; // struct flock unlock = { F_UNLCK, SEEK_SET, pos_, sizeof(Record), }; // SYSCALL(::fcntl(fd_,F_SETLK,&lock)); SYSCALL(size = ::write(fd_, &buffer_, sizeof(Record))); // SYSCALL(::fcntl(fd_,F_SETLK,&unlock)); pos_ += size; ASSERT(size == sizeof(Record)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/MoverTransfer.h0000664000175000017500000000405615161702250020252 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File MoverTransfer.h // Baudouin Raoult - (c) ECMWF Jun 11 #ifndef eckit_MoverTransfer_h #define eckit_MoverTransfer_h #include "eckit/io/DataHandle.h" #include "eckit/io/Length.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class TransferWatcher; class MoverTransfer { public: // -- Exceptions // None // -- Contructors MoverTransfer(TransferWatcher& = TransferWatcher::dummy()); // -- Destructor ~MoverTransfer(); // -- Convertors // None // -- Operators // None // -- Methods Length transfer(DataHandle&, DataHandle&); TransferWatcher& watcher() const { return watcher_; } // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods // void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed MoverTransfer(const MoverTransfer&); MoverTransfer& operator=(const MoverTransfer&); // -- Members // None TransferWatcher& watcher_; // -- Methods // None // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends // friend std::ostream& operator<<(std::ostream& s,const MoverTransfer& p) // { p.print(s); return s; } }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/StdFile.h0000664000175000017500000000312315161702250017001 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 96 #ifndef eckit_io_StdFile_h #define eckit_io_StdFile_h #include #include "eckit/filesystem/PathName.h" namespace eckit { class PathName; /// Wrapper around a stdio FILE* /// Use this for class members class StdFile { public: StdFile(const PathName& name, const std::string& mode = "r"); StdFile(const StdFile&) = delete; StdFile& operator=(const StdFile&) = delete; StdFile(StdFile&&) = delete; StdFile& operator=(StdFile&&) = delete; /// @pre must have been closed ~StdFile(); /// Get the FILE* but don't call fclose on it operator FILE*() { return file_; } bool isOpen() { return file_; } /// @throws on fclose failure void close() noexcept(false); private: // members FILE* file_; }; /// Wrapper around a stdio FILE* /// Use this for stack objects that automatically close class AutoStdFile : public StdFile { void close() { StdFile::close(); } public: AutoStdFile(const PathName& name, const std::string& mode = "r") : StdFile(name, mode) {} ~AutoStdFile() noexcept(false) { close(); } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/StdPipe.h0000664000175000017500000000301215161702250017014 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 1996 #ifndef eckit_io_StdPipe_h #define eckit_io_StdPipe_h #include #include #include "eckit/io/AutoCloser.h" namespace eckit { /// Simple wrapper for pipes class StdPipe { public: StdPipe(const std::string& name, const std::string& mode = "r"); StdPipe(const StdPipe&) = delete; StdPipe& operator=(const StdPipe&) = delete; StdPipe(StdPipe&&) = delete; StdPipe& operator=(StdPipe&&) = delete; /// @pre must have been closed ~StdPipe(); /// Get the FILE* but don't call fclose on it operator FILE*() { return file_; } bool isOpen() { return file_; } /// @throws on fclose failure void close() noexcept(false); private: // members FILE* file_; }; /// Wrapper around a stdio FILE* /// Use this for stack objects that automatically close class AutoStdPipe : public StdPipe { public: AutoStdPipe(const std::string& name, const std::string& mode = "r") : StdPipe(name, mode) {} ~AutoStdPipe() noexcept(false) { close(); } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/TCPHandle.h0000664000175000017500000000352715161702250017221 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/TCPHandle.h // Baudouin Raoult - ECMWF Jul 96 #ifndef eckit_filesystem_TCPHandle_h #define eckit_filesystem_TCPHandle_h #include "eckit/io/DataHandle.h" #include "eckit/net/TCPClient.h" namespace eckit { class TCPHandle : public DataHandle { public: // -- Contructors TCPHandle(Stream&); TCPHandle(const std::string& host, int port); // -- Destructor ~TCPHandle(); // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void rewind() override; DataHandle* clone() const override; void print(std::ostream&) const override; std::string title() const override; bool moveable() const override { return true; } bool canSeek() const override { return false; } virtual void selectMover(eckit::MoverTransferSelection&, bool) const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } protected: std::string host_; int port_; net::TCPClient connection_; private: static ClassSpec classSpec_; static Reanimator reanimator_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/ResizableBuffer.h0000664000175000017500000000132515161702250020523 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_io_ResizableBuffer_h #define eckit_io_ResizableBuffer_h #warning \ "Header eckit/io/ResizableBuffer.h and class eckit::ResizableBuffer is deprecated -- include eckit/io/Buffer.h and use eckit::Buffer" #include "eckit/io/Buffer.h" namespace eckit { using ResizableBuffer = Buffer; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/CircularBuffer.cc0000664000175000017500000000615415161702250020512 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/CircularBuffer.h" #include "eckit/maths/Functions.h" #include "eckit/thread/AutoLock.h" namespace eckit { CircularBuffer::CircularBuffer(size_t size, size_t capacity) : buffer_(new char[size]), increment_(size), size_(size), capacity_(capacity), pos_(0), used_(0) { ASSERT(buffer_); ASSERT(size_ <= capacity_); } CircularBuffer::~CircularBuffer() { delete[] buffer_; } size_t CircularBuffer::write(const void* buffer, size_t length) { AutoLock lock(mutex_); size_t left = size_ - used_; // std::cout << "CircularBuffer::write(" << length << ") left = " << left << std::endl; // std::cout << "Pos " << pos_ << ", used " << used_ << std::endl; if (length > left) { size_t newsize = eckit::round(size_ + length, increment_); // std::cout << "CircularBuffer::resize(" << size_ << " => " << newsize << std::endl; if (newsize > capacity_) { std::ostringstream oss; oss << "CircularBuffer: cannot grow beyound capacity of " << capacity_ << " bytes"; throw eckit::SeriousBug(oss.str()); } // std::cout << "Newsize " << newsize<< std::endl; char* buffer = new char[newsize]; ASSERT(buffer); size_t save = used_; ASSERT(read(buffer, save) == save); pos_ = 0; used_ = save; size_ = newsize; delete[] buffer_; buffer_ = buffer; } const char* p = static_cast(buffer); size_t start = (pos_ + used_) % size_; size_t n = std::min(size_ - start, size_t(length)); ::memcpy(buffer_ + start, p, n); ::memcpy(buffer_, p + n, length - n); used_ += length; return length; } size_t CircularBuffer::read(void* buffer, size_t length) { AutoLock lock(mutex_); // std::cout << "CircularBuffer::read(" << length << ") used = " << used_ << std::endl; size_t len = std::min(used_, length); // std::cout << "Len " << len << std::endl; char* p = static_cast(buffer); size_t n = std::min(size_ - pos_, len); // std::cout << "Pos " << pos_ << " n " << n << std::endl; ::memcpy(p, buffer_ + pos_, n); ::memcpy(p + n, buffer_, len - n); pos_ = (pos_ + len) % size_; used_ -= len; return len; } size_t CircularBuffer::length() const { AutoLock lock(mutex_); return used_; } void CircularBuffer::clear() { AutoLock lock(mutex_); pos_ = 0; used_ = 0; } size_t CircularBuffer::capacity() const { AutoLock lock(mutex_); return capacity_; } size_t CircularBuffer::size() const { AutoLock lock(mutex_); return size_; } } // namespace eckit eckit-2.0.7/src/eckit/io/Select.h0000664000175000017500000000263315161702250016673 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Mar 97 #ifndef eckit_Select_h #define eckit_Select_h #include namespace eckit { //----------------------------------------------------------------------------- namespace net { class TCPSocket; }; /// Wraps calls to select class Select { public: // -- Contructors Select(); explicit Select(int); explicit Select(net::TCPSocket&); Select(const Select&) = delete; Select& operator=(const Select&) = delete; Select(Select&&) = delete; Select& operator=(Select&&) = delete; // -- Destructor ~Select(); // -- Methods bool ready(long sec = 20); void add(net::TCPSocket&); void add(int); void remove(net::TCPSocket&); void remove(int); bool set(net::TCPSocket&); bool set(int); private: // -- Members fd_set files_; fd_set set_; int last_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/BufferList.cc0000664000175000017500000000325415161702250017657 0ustar alastairalastair/* * (C) Copyright 2021- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/BufferList.h" #include #include #include "eckit/io/Offset.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- void BufferList::append(Buffer&& buf) { buffers_.emplace_back(std::move(buf)); } Length BufferList::size() const { return std::accumulate(buffers_.begin(), buffers_.end(), Length(0), [](const Length& lhs, const Buffer& rhs) { return lhs + Length(rhs.size()); }); } Buffer BufferList::consolidate() { const size_t nbuffs = count(); if (nbuffs == 0) { return Buffer(); } // optimize for count() = 1 // we can do this optimisation because consolidate() clears the contents of the list if (nbuffs == 1) { Buffer b = std::move(buffers_.front()); buffers_.clear(); return b; } Buffer result(size()); Offset offset = 0; for (const auto& buffer : buffers_) { result.copy(buffer, buffer.size(), offset); offset += buffer.size(); } buffers_.clear(); // deallocate all buffers return result; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/SockBuf.cc0000664000175000017500000000373115161702250017146 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/SockBuf.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- SockBuf::SockBuf(net::TCPSocket& proto) : protocol_(proto) { #ifndef OLD_STREAMBUF /* setg(in_, in_, in_ + sizeof(in_) ); */ setg(in_, in_, in_); setp(out_, out_ + sizeof(out_)); #else setb(in_, in_ + sizeof(in_), 0); setg(in_, in_, in_); setp(out_, out_ + sizeof(out_)); #endif } SockBuf::~SockBuf() { sync(); } int SockBuf::sync() { if (!protocol_.isConnected()) { setp(pbase(), epptr()); return EOF; } if (protocol_.write(pbase(), pptr() - pbase()) < 0) { protocol_.close(); return EOF; } setp(pbase(), epptr()); return 0; } int SockBuf::overflow(int c) { if (sync()) { return EOF; } if (c == EOF) { return 0; } sputc(c); return 0; } int SockBuf::underflow() { if (gptr() < egptr()) { return *(unsigned char*)gptr(); } if (!protocol_.isConnected()) { return EOF; } #ifndef OLD_STREAMBUF int n = protocol_.read(in_, sizeof(in_)); #else int n = protocol_.read(base(), sizeof(in_)); #endif if (n == EOF || n == 0) { protocol_.close(); return EOF; } #ifndef OLD_STREAMBUF setg(in_, in_, in_ + n); #else setg(eback(), base(), base() + n); #endif return *(unsigned char*)gptr(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/PooledFile.cc0000664000175000017500000001641115161702250017633 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/PooledFile.h" #include #include #include #include #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/log/Bytes.h" namespace eckit { class PoolFileEntry; } namespace { class Pool { private: Pool() {} std::map> filePool_; std::mutex filePoolMutex_; public: static Pool& instance() { static Pool pool; return pool; } eckit::PoolFileEntry* get(const eckit::PathName& name); void erase(const eckit::PathName& name); }; } // namespace namespace eckit { struct PoolFileEntryStatus { off_t position_; bool opened_; PoolFileEntryStatus() : position_(0), opened_(false) {} }; class PoolFileEntry { public: std::string name_; FILE* file_; size_t count_; std::unique_ptr buffer_; std::map statuses_; size_t nbOpens_ = 0; size_t nbReads_ = 0; size_t nbSeeks_ = 0; public: PoolFileEntry(const std::string& name) : name_(name), file_{nullptr}, count_(0) {} void doClose() { if (file_) { Log::debug() << "Closing from file " << name_ << std::endl; if (::fclose(file_) != 0) { throw PooledFileError(name_, "Failed to close", Here()); } file_ = nullptr; buffer_.reset(); } } void add(const PooledFile* file) { ASSERT(statuses_.find(file) == statuses_.end()); statuses_[file] = PoolFileEntryStatus(); } void remove(const PooledFile* file) { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); statuses_.erase(s); if (statuses_.size() == 0) { doClose(); Pool::instance().erase(name_); // No code after !!! } } void open(const PooledFile* file) { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); ASSERT(!s->second.opened_); if (!file_) { nbOpens_++; file_ = ::fopen(name_.c_str(), "r"); if (!file_) { throw PooledFileError(name_, "Failed to open", Here()); } Log::debug() << "PooledFile::openForRead " << name_ << std::endl; static size_t bufferSize = Resource("FileHandleIOBufferSize;$FILEHANDLE_IO_BUFFERSIZE;-FileHandleIOBufferSize", 0); if (bufferSize) { Log::debug() << "PooledFile using " << Bytes(bufferSize) << std::endl; buffer_.reset(new Buffer(bufferSize)); Buffer& b = *(buffer_.get()); ::setvbuf(file_, b, _IOFBF, bufferSize); } } s->second.opened_ = true; s->second.position_ = 0; } void close(const PooledFile* file) { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); ASSERT(s->second.opened_); s->second.opened_ = false; } int fileno(const PooledFile* file) const { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); ASSERT(s->second.opened_); return ::fileno(file_); } long read(const PooledFile* file, void* buffer, long len) { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); ASSERT(s->second.opened_); if (::fseeko(file_, s->second.position_, SEEK_SET) < 0) { throw PooledFileError(name_, "Failed to seek", Here()); } // Log::debug() < "Reading @ position " << s->second.position_ << " file : " << name_ << // std::endl; size_t length = size_t(len); size_t n = ::fread(buffer, 1, length, file_); if (n != length && ::ferror(file_)) { throw PooledFileError(name_, "Read error", Here()); } s->second.position_ = ::ftello(file_); nbReads_++; return n; } long seek(const PooledFile* file, off_t position) { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); ASSERT(s->second.opened_); if (::fseeko(file_, position, SEEK_SET) != 0) { std::ostringstream s; s << name_ << ": cannot seek to " << position << " (file=" << ::fileno(file_) << ")"; throw ReadError(s.str()); } s->second.position_ = ::ftello(file_); ASSERT(s->second.position_ == position); nbSeeks_++; return s->second.position_; } long seekEnd(const PooledFile* file) { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); ASSERT(s->second.opened_); if (::fseeko(file_, 0, SEEK_END) != 0) { std::ostringstream s; s << name_ << ": cannot seek to end (file=" << ::fileno(file_) << ")"; throw ReadError(s.str()); } s->second.position_ = ::ftello(file_); nbSeeks_++; return s->second.position_; } }; PooledFile::PooledFile(const PathName& name) : name_(name), entry_(Pool::instance().get(name)) { entry_->add(this); } PooledFile::~PooledFile() { ASSERT(entry_); entry_->remove(this); } void PooledFile::open() { ASSERT(entry_); entry_->open(this); } void PooledFile::close() { ASSERT(entry_); entry_->close(this); } off_t PooledFile::seek(off_t offset) { ASSERT(entry_); return entry_->seek(this, offset); } off_t PooledFile::seekEnd() { ASSERT(entry_); return entry_->seekEnd(this); } off_t PooledFile::rewind() { return seek(0); } int PooledFile::fileno() const { ASSERT(entry_); return entry_->fileno(this); } size_t PooledFile::nbOpens() const { ASSERT(entry_); return entry_->nbOpens_; } size_t PooledFile::nbReads() const { ASSERT(entry_); return entry_->nbReads_; } size_t PooledFile::nbSeeks() const { ASSERT(entry_); return entry_->nbSeeks_; } long PooledFile::read(void* buffer, long len) { ASSERT(entry_); return entry_->read(this, buffer, len); } PooledFileError::PooledFileError(const std::string& file, const std::string& msg, const CodeLocation& loc) : FileError(msg + " : error on pooled file " + file, loc) {} } // namespace eckit namespace { eckit::PoolFileEntry* Pool::get(const eckit::PathName& name) { std::lock_guard lock(filePoolMutex_); auto j = filePool_.find(name); if (j == filePool_.end()) { filePool_.emplace(name, new eckit::PoolFileEntry(name)); j = filePool_.find(name); } return (*j).second.get(); } void Pool::erase(const eckit::PathName& name) { std::lock_guard lock(filePoolMutex_); filePool_.erase(name); } } // namespace eckit-2.0.7/src/eckit/io/HandleHolder.h0000664000175000017500000000242615161702250020005 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // Baudouin Raoult - ECMWF Dec 2013 #ifndef eckit_filesystem_HandleHolder_h #define eckit_filesystem_HandleHolder_h #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/log/Timer.h" #include "eckit/types/Types.h" namespace eckit { //----------------------------------------------------------------------------- class HandleHolder { public: // -- Contructors HandleHolder(DataHandle& handle); HandleHolder(DataHandle* handle); // -- Destructor ~HandleHolder(); // -- Methods // -- Overridden methods protected: DataHandle& handle() { return *handle_; } const DataHandle& handle() const { return *handle_; } private: // -- Members DataHandle* handle_; bool owned_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/Offset.h0000664000175000017500000000567015161702250016706 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Offset.h // Baudouin Raoult - ECMWF Jul 96 #ifndef eckit_Offset_h #define eckit_Offset_h #include #include #include "eckit/io/Length.h" #include "eckit/persist/Bless.h" namespace eckit { //----------------------------------------------------------------------------- // Forwarded declarations class Bless; class Stream; // But because the compiler aligns long longs // on 64bits boundaries and longs on 32 bits boundaries, // we need the help of a little pragma here, to make ObjectStore happy #ifdef _AIX #pragma options align = twobyte #endif class Offset { public: // types using value_t = long long; public: friend std::ostream& operator<<(std::ostream& s, const Offset& x); friend Stream& operator<<(Stream& s, const Offset& x); friend Stream& operator>>(Stream& s, Offset& x); // Offset(fpos_t); <- To implement Offset(value_t l = 0) : value_(l) {} Offset(const Offset& other) : value_(other.value_) {} #include "eckit/io/Offset.b" Offset& operator=(const Offset& other) { value_ = other.value_; return *this; } bool operator==(const Offset& other) const { return value_ == other.value_; } bool operator!=(const Offset& other) const { return value_ != other.value_; } bool operator<(const Offset& other) const { return value_ < other.value_; } bool operator>(const Offset& other) const { return value_ > other.value_; } bool operator<=(const Offset& other) const { return value_ <= other.value_; } bool operator>=(const Offset& other) const { return value_ >= other.value_; } Offset operator+(const Length& other) const { return Offset(value_ + other.value_); } void operator+=(const Length& other) { value_ += other.value_; } void operator-=(const Length& other) { value_ -= other.value_; } Length operator-(const Offset& other) const { return value_ - other.value_; } operator value_t() const { return value_; } void dump(DumpLoad&) const; void load(DumpLoad&); private: // -- Members value_t value_; friend class Length; }; using OffsetList = std::vector; //----------------------------------------------------------------------------- // Global routines // Sort both std::vector according to offset void sort(OffsetList&, LengthList&); bool compress(OffsetList&, LengthList&); void accumulate(const LengthList&, OffsetList&, const Offset& = 0); #ifdef _AIX #pragma options align = reset #endif //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/URLHandle.h0000664000175000017500000000432715161702250017234 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date June 2017 #ifndef eckit_filesystem_URLHandle_h #define eckit_filesystem_URLHandle_h #include #include "eckit/exception/Exceptions.h" #include "eckit/io/DataHandle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class URLException : public Exception { int code_; public: URLException(const std::string& what, int code) : Exception(what), code_(code) {} int code() const { return code_; } }; //---------------------------------------------------------------------------------------------------------------------- class URLHandle : public DataHandle { public: URLHandle(const std::string& uri, bool useSSL = true); URLHandle(Stream&); ~URLHandle(); // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; // void rewind() override; void print(std::ostream&) const override; Length estimate() override; Length size() override; bool canSeek() const override { return false; } // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: std::string uri_; std::unique_ptr handle_; bool useSSL_; DataHandle& handle(); static ClassSpec classSpec_; static Reanimator reanimator_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/FileDescHandle.cc0000664000175000017500000000377415161702250020413 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/FileDescHandle.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- void FileDescHandle::print(std::ostream& s) const { s << "FileDescHandle[fd=" << fd_ << ']'; } void FileDescHandle::encode(Stream& s) const { NOTIMP; } FileDescHandle::FileDescHandle(int fd, bool close) : fd_(fd), close_(close) {} FileDescHandle::~FileDescHandle() { if (fd_ != -1) { close(); } } Length FileDescHandle::openForRead() { return 0; } void FileDescHandle::openForWrite(const Length&) {} void FileDescHandle::openForAppend(const Length&) {} long FileDescHandle::read(void* buffer, long length) { return ::read(fd_, buffer, length); } long FileDescHandle::write(const void* buffer, long length) { return ::write(fd_, buffer, length); } void FileDescHandle::close() { if (close_ && (fd_ != -1)) { SYSCALL(::close(fd_)); fd_ = -1; } } Offset FileDescHandle::position() { off_t pos; SYSCALL(pos = ::lseek(fd_, 0, SEEK_CUR)); return pos; } Offset FileDescHandle::seek(const Offset& o) { off_t pos; SYSCALL(pos = ::lseek(fd_, o, SEEK_SET)); return pos; } void FileDescHandle::skip(const Length& l) { SYSCALL(::lseek(fd_, l, SEEK_CUR)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/Compress.cc0000664000175000017500000001726515161702250017414 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/BitIO.h" #include "eckit/io/Compress.h" #include "eckit/io/DataHandle.h" // This code is written for readibility, not speed // See https://users.cs.cf.ac.uk/Dave.Marshall/Multimedia/node214.html // for a description of the algorythm // and https://www.cs.duke.edu/csed/curious/compression/lzw.html namespace eckit { namespace { inline static size_t MAX_CODE(size_t nbits) { return (1 << nbits) - 1; } enum { RESET_TABLE = 256, END_MARKER = 257, FIRST_CODE = 258, MIN_BITS = 9, }; /* UNUSED static void print_code(std::ostream& out, size_t s) { switch (s) { case RESET_TABLE: out << "(reset)"; break; case END_MARKER: out << "(end)"; break; default: if (::isprint(s)) { out << char(s); } else { if (s < 256) { out << std::hex << std::setfill('0') << std::setw(2) << s << std::dec << std::setfill(' '); } else { out << '(' << s << ')'; } } break; } } */ class Entry { std::vector chars_; size_t code_; public: Entry(size_t = END_MARKER); Entry operator+(unsigned char) const; Entry& operator=(unsigned char); bool empty() const; void code(size_t c) { code_ = c; } unsigned char firstChar() const { ASSERT(chars_.size()); return chars_[0]; } bool operator<(const Entry& other) const { return chars_ < other.chars_; } /* unused */ // friend std::ostream& operator<<(std::ostream& out, const Entry& e) { // e.print(out); // return out; // } /* unused */ // void Entry::print(std::ostream& out) const { // out << '['; // print_code(out, code_); // out << " -> "; // for(size_t i = 0; i < chars_.size(); ++i) { // print_code(out, chars_[i]); // } // out << ']'; // } void output(eckit::BitIO& out) const; void output(eckit::BitIO& out, size_t nbits) const; }; Entry::Entry(size_t code) : code_(code) { if (code < 256) { chars_.push_back(code); } } Entry Entry::operator+(unsigned char c) const { Entry result = *this; result.chars_.push_back(c); return result; } Entry& Entry::operator=(unsigned char c) { chars_.clear(); chars_.push_back(c); code_ = c; return *this; } void Entry::output(eckit::BitIO& out) const { // std::cout << "Output " << *this << std::endl; for (size_t i = 0; i < chars_.size(); ++i) { out.write(chars_[i], 8); } } bool Entry::empty() const { return chars_.empty(); } void Entry::output(eckit::BitIO& out, size_t nbits) const { // std::cout << "Send " // << *this // << " (nbits=" // << nbits // << ")" // << std::endl; ASSERT(code_ <= MAX_CODE(nbits)); out.write(code_, nbits); } //---------------------------------------------------------------------------------------------------------------------- static inline size_t next_byte(BitIO& in) { size_t byte = in.read(8, END_MARKER); // std::cout << "Read "; // print_code(std::cout, byte); // std::cout << std::endl; return byte; } static void init_table(std::set& table) { table.clear(); for (size_t c = 0; c < 256; ++c) { table.insert(Entry(c)); } } } // namespace Compress::Compress(size_t maxBits) : maxBits_(maxBits) {} size_t Compress::encode(DataHandle& in, DataHandle& out) { Entry eoi(END_MARKER); Entry reset(RESET_TABLE); std::set code_table; init_table(code_table); BitIO bin(in); BitIO bout(out); size_t nbits = MIN_BITS; size_t next_code = FIRST_CODE; size_t max_code = MAX_CODE(nbits); reset.output(bout, nbits); Entry w; for (;;) { size_t k = next_byte(bin); if (k == END_MARKER) { break; } Entry wk = w + k; std::set::iterator j = code_table.find(wk); // Sequence in table if (j != code_table.end()) { // '*j' is the same as 'wp' // but should contain a valid code w = *j; } else { w.output(bout, nbits); wk.code(next_code++); code_table.insert(wk); w = k; if (next_code >= max_code) { if (nbits == maxBits_) { reset.output(bout, nbits); nbits = MIN_BITS; max_code = MAX_CODE(nbits); next_code = FIRST_CODE; init_table(code_table); } else { nbits++; max_code = MAX_CODE(nbits); } } } } if (!w.empty()) { w.output(bout, nbits); } eoi.output(bout, nbits); return bout.byteCount(); } //---------------------------------------------------------------------------------------------------------------------- static void init_table(std::map& table) { table.clear(); for (size_t i = 0; i < 256; ++i) { table[i] = Entry(i); } } size_t Compress::decode(DataHandle& in, DataHandle& out) { std::map table; init_table(table); BitIO bin(in); BitIO bout(out); size_t nbits = MIN_BITS; Entry w; size_t next_code = FIRST_CODE; size_t max_code = MAX_CODE(nbits) - 1; for (;;) { if (next_code >= max_code) { nbits = std::min(maxBits_, nbits + 1); max_code = MAX_CODE(nbits) - 1; // std::cout << "DECODE nbits " << nbits << std::endl; } size_t k = bin.read(nbits, END_MARKER); // std::cout << "Got code "; // print_code(std::cout, k); // std::cout << std::endl; if (k == END_MARKER) { break; } /* This should be the first code */ if (k == RESET_TABLE) { // std::cout << "RESET_TABLE" << std::endl; nbits = MIN_BITS; max_code = MAX_CODE(nbits) - 1; next_code = FIRST_CODE; init_table(table); k = bin.read(nbits, END_MARKER); if (k == END_MARKER) { break; } // std::cout << "Got code "; // print_code(std::cout, k); // std::cout << std::endl; w = k; w.output(bout); continue; } Entry e; std::map::iterator j = table.find(k); if (j != table.end()) { e = (*j).second; } else { ASSERT(k == next_code); e = w + w.firstChar(); } e.output(bout); Entry n = w + e.firstChar(); n.code(next_code); table[next_code] = n; next_code++; // std::cout << "Add code " << n << std::endl; w = e; } return bout.byteCount(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/FTPHandle.cc0000664000175000017500000000646415161702250017365 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/io/FTPHandle.h" #include "eckit/log/Log.h" #include "eckit/net/IPAddress.h" #include "eckit/net/TCPServer.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec FTPHandle::classSpec_ = { &DataHandle::classSpec(), "FTPHandle", }; Reanimator FTPHandle::reanimator_; const char* FTPHandle::FTPError::what() const noexcept { return "FTP Error"; } void FTPHandle::print(std::ostream& s) const { s << "FTPHandle[file=" << remote_ << ",host=" << host_ << ",port=" << port_ << ']'; } void FTPHandle::encode(Stream& s) const { DataHandle::encode(s); s << remote_; s << host_; s << port_; } FTPHandle::FTPHandle(Stream& s) : DataHandle(s), port_(0) { s >> remote_; s >> host_; s >> port_; } std::string FTPHandle::readLine() { std::string s; char c; while (cmds_.read(&c, 1) == 1 && c != '\n') { s += c; } Log::info() << "receive " << s << std::endl; return s; } void FTPHandle::ftpCommand(const std::string& s) { Log::info() << "send " << s << std::endl; cmds_.write(s.c_str(), s.length()); cmds_.write("\r\n", 2); std::string out = readLine(); if (atoi(out.c_str()) / 100 == 5) { throw FTPError(); } } FTPHandle::FTPHandle(const std::string& host, const std::string& remote, int port) : remote_(remote), host_(host), port_(port) {} void FTPHandle::open(const std::string& cmd) { cmds_.connect(host_, port_); readLine(); ftpCommand("USER mab"); ftpCommand("PASS 04clave"); ftpCommand("TYPE I"); net::EphemeralTCPServer server; int port = htons(server.localPort()); std::string addr = net::IPAddress(server.localAddr()).asString(); char p[1024]; snprintf(p, 1024, "PORT %s,%d,%d", addr.c_str(), port / 256, port % 256); char* q = p; while (*q) { if (*q == '.') { *q = ','; } q++; } ftpCommand(p); snprintf(p, 1024, "%s %s", cmd.c_str(), remote_.c_str()); ftpCommand(p); data_ = server.accept(); } Length FTPHandle::openForRead() { open("RETR"); return 0; } void FTPHandle::openForWrite(const Length&) { open("STOR"); } void FTPHandle::openForAppend(const Length&) { NOTIMP; } long FTPHandle::read(void* buffer, long length) { return data_.read(buffer, length); } long FTPHandle::write(const void* buffer, long length) { return data_.write(buffer, length); } void FTPHandle::close() { data_.close(); readLine(); ftpCommand("QUIT"); cmds_.close(); } void FTPHandle::rewind() { NOTIMP; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/PooledHandle.cc0000664000175000017500000001514615161702250020153 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/PooledHandle.h" #include "eckit/utils/MD5.h" namespace eckit { class PoolHandleEntry; /// @note in anonymous namespace to solve some compilers link issue (eg. xlc) namespace { static thread_local std::map> pool_; } static size_t maxPooledHandles() { static int maxPooledHandles = eckit::Resource("$ECKIT_MAX_POOLED_HANDLES;maxPooledHandles", 128); return size_t(maxPooledHandles); } struct PoolHandleEntryStatus { Offset position_; bool opened_; PoolHandleEntryStatus() : position_(0), opened_(false) {} }; class PoolHandleEntry { public: PathName path_; std::unique_ptr handle_; Length estimate_; size_t count_; std::map statuses_; size_t nbOpens_ = 0; size_t nbReads_ = 0; size_t nbSeeks_ = 0; size_t nbCloses_ = 0; public: explicit PoolHandleEntry(const PathName& path) : path_(path), handle_{nullptr}, count_(0) {} ~PoolHandleEntry() { LOG_DEBUG_LIB(LibEcKit) << *this << std::endl; } friend std::ostream& operator<<(std::ostream& s, const PoolHandleEntry& e) { e.print(s); return s; } void print(std::ostream& s) const { s << "PoolHandleEntry[" << path_ << ",opens=" << nbOpens_ << ",reads=" << nbReads_ << ",seeks=" << nbSeeks_ << ",closes=" << nbCloses_ << "]"; } void doClose() { if (handle_) { LOG_DEBUG_LIB(LibEcKit) << "PooledHandle::close(" << *handle_ << ")" << std::endl; handle_->close(); handle_.reset(); } } void add(const PooledHandle* file) { ASSERT(statuses_.find(file) == statuses_.end()); statuses_[file] = PoolHandleEntryStatus(); } void remove(const PooledHandle* file) { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); statuses_.erase(s); if (statuses_.size() == 0) { doClose(); pool_.erase(path_); // No code after !!! } } Length open(const PooledHandle* file) { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); ASSERT(!s->second.opened_); if (!handle_) { checkMaxPooledHandles(); nbOpens_++; handle_.reset(path_.fileHandle()); ASSERT(handle_); LOG_DEBUG_LIB(LibEcKit) << "PooledHandle::openForRead(" << *handle_ << ")" << std::endl; estimate_ = handle_->openForRead(); } s->second.opened_ = true; s->second.position_ = 0; return estimate_; } bool canClose() { for (auto i = statuses_.begin(); i != statuses_.end(); ++i) { if ((*i).second.opened_) { return false; } } return true; } void checkMaxPooledHandles() { size_t opened = 0; for (auto i = pool_.begin(); i != pool_.end(); ++i) { if ((*i).second->handle_) { opened++; } } if (opened >= maxPooledHandles()) { LOG_DEBUG_LIB(LibEcKit) << "PooledHandle maximum number of open files reached: " << maxPooledHandles() << std::endl; for (auto i = pool_.begin(); i != pool_.end(); ++i) { if ((*i).second->canClose()) { (*i).second->doClose(); } } } } void close(const PooledHandle* file) { auto s = statuses_.find(file); ASSERT(s != statuses_.end()); ASSERT(s->second.opened_); s->second.opened_ = false; nbCloses_++; } long read(const PooledHandle* handle, void* buffer, long len) { auto s = statuses_.find(handle); ASSERT(s != statuses_.end()); ASSERT(s->second.opened_); ASSERT(handle_->seek(s->second.position_) == s->second.position_); long n = handle_->read(buffer, len); s->second.position_ = handle_->position(); nbReads_++; return n; } long seek(const PooledHandle* handle, Offset position) { auto s = statuses_.find(handle); ASSERT(s != statuses_.end()); ASSERT(s->second.opened_); ASSERT(handle_->seek(position) == position); s->second.position_ = handle_->position(); ASSERT(s->second.position_ == position); nbSeeks_++; return s->second.position_; } }; PooledHandle::PooledHandle(const PathName& path) : path_(path), entry_{nullptr} { auto j = pool_.find(path); if (j == pool_.end()) { pool_.emplace(std::make_pair(path, std::unique_ptr(new PoolHandleEntry(path)))); j = pool_.find(path); } entry_ = (*j).second.get(); entry_->add(this); } PooledHandle::~PooledHandle() { ASSERT(entry_); entry_->remove(this); } void PooledHandle::print(std::ostream& s) const { s << "PooledHandle[" << path_ << "]"; } Length PooledHandle::openForRead() { ASSERT(entry_); return entry_->open(this); } void PooledHandle::openForWrite(const Length&) { NOTIMP; } void PooledHandle::openForAppend(const Length&) { NOTIMP; } long PooledHandle::write(const void*, long) { NOTIMP; } void PooledHandle::close() { ASSERT(entry_); entry_->close(this); } Offset PooledHandle::seek(const Offset& offset) { ASSERT(entry_); return entry_->seek(this, offset); } size_t PooledHandle::nbOpens() const { ASSERT(entry_); return entry_->nbOpens_; } size_t PooledHandle::nbReads() const { ASSERT(entry_); return entry_->nbReads_; } size_t PooledHandle::nbSeeks() const { ASSERT(entry_); return entry_->nbSeeks_; } long PooledHandle::read(void* buffer, long len) { ASSERT(entry_); return entry_->read(this, buffer, len); } void PooledHandle::hash(MD5& md5) const { md5 << "PooledHandle"; md5 << std::string(entry_->path_); } Offset PooledHandle::position() { return entry_->handle_->position(); } } // namespace eckit eckit-2.0.7/src/eckit/io/BufferList.h0000664000175000017500000000305115161702250017514 0ustar alastairalastair/* * (C) Copyright 2021- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @author Tiago Quintino /// @date March 2021 #pragma once #include #include "eckit/io/Buffer.h" #include "eckit/io/Length.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// A class to aggregate buffers into a single object that can be read as a whole class BufferList { public: // methods BufferList() = default; BufferList(const BufferList&) = delete; BufferList& operator=(const BufferList&) = delete; BufferList(BufferList&&) = default; BufferList& operator=(BufferList&&) = default; ~BufferList() = default; void append(Buffer&& buf); size_t count() const { return buffers_.size(); } Length size() const; /// @note After consolidation the internal list is cleared and memory is deallocated /// @post count() == 0 and size() == 0 Buffer consolidate(); private: // members std::list buffers_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/AIOHandle.cc0000664000175000017500000002135515161702250017340 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit.h" #include #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/AIOHandle.h" #include "eckit/log/Log.h" #include "eckit/maths/Functions.h" #include "eckit/memory/Zero.h" #include "eckit/os/Stat.h" #include "eckit/types/Types.h" #if eckit_HAVE_AIO #include #endif namespace eckit { //---------------------------------------------------------------------------------------------------------------------- #if eckit_HAVE_AIO struct AIOBuffer { public: // methods explicit AIOBuffer() { eckit::zero(aio_); } AIOBuffer(const AIOBuffer&) = delete; AIOBuffer& operator=(const AIOBuffer&) = delete; AIOBuffer(AIOBuffer&&) = delete; AIOBuffer& operator=(AIOBuffer&&) = delete; ~AIOBuffer() { delete buff_; } void resize(size_t sz) { if (buff_ == nullptr || buff_->size() < sz) { delete buff_; buff_ = new Buffer(eckit::round(sz, 4 * 1024)); } ASSERT(buff_ && buff_->size() >= sz); } void write(int fd, off_t pos, const void* buffer, size_t length) { resize(length); ::memcpy(buff_->data(), buffer, length); len_ = length; zero(aio_); aio_.aio_fildes = fd; aio_.aio_offset = pos; aio_.aio_buf = buff_->data(); aio_.aio_nbytes = length; aio_.aio_sigevent.sigev_notify = SIGEV_NONE; caioptr_ = &aio_; SYSCALL(::aio_write(&aio_)); active_ = true; } struct aiocb* aioptr() { return &aio_; } const struct aiocb* caioptr() const { return caioptr_; } bool active() const { return active_; } void active(bool v) { active_ = v; } size_t length() const { return len_; } private: // members aiocb aio_; const aiocb* caioptr_ = nullptr; eckit::Buffer* buff_ = nullptr; size_t len_ = 0; bool active_ = false; }; //---------------------------------------------------------------------------------------------------------------------- AIOHandle::AIOHandle(const PathName& path, size_t count, size_t /* buffsize */, bool fsync) : path_(path), used_(0), count_(count), fd_(-1), pos_(0), fsync_(fsync) { #ifdef AIO_LISTIO_MAX count_ = std::min(count_, AIO_LISTIO_MAX); #endif #ifdef AIO_MAX count_ = std::min(count_, AIO_MAX); #endif buffers_.reserve(count_); for (size_t i = 0; i < count_; i++) { buffers_.push_back(new AIOBuffer()); } } size_t AIOHandle::getFreeSlot() { // when starting we just take a new until we allocate all new slots if (used_ < count_) { return used_++; } size_t n = 0; std::vector aioplist; std::transform(buffers_.begin(), buffers_.end(), std::back_inserter(aioplist), [](AIOBuffer* b) { return b->caioptr(); }); /* wait until at least one buffer is done */ errno = 0; while (::aio_suspend(aioplist.data(), aioplist.size(), nullptr) < 0) { if (errno != EINTR) { throw FailedSystemCall("aio_suspend"); } } // find which one has completed bool ok = false; for (n = 0; n < count_; n++) { int e = ::aio_error(buffers_[n]->caioptr()); if (e == EINPROGRESS) { continue; } buffers_[n]->active(false); if (e == 0) { ssize_t len = ::aio_return(buffers_[n]->aioptr()); if (len != buffers_[n]->length()) { // TODO: retry when filesystems are full std::ostringstream os; os << "AIOHandle: only " << len << " bytes written instead of " << buffers_[n]->length(); throw WriteError(os.str()); } ok = true; break; } throw FailedSystemCall("aio_error"); } ASSERT(ok); return n; } long AIOHandle::write(const void* buffer, long length) { if (length == 0) { return 0; } size_t n = getFreeSlot(); buffers_[n]->write(fd_, pos_, buffer, (size_t)length); pos_ += length; return length; } void AIOHandle::flush() { bool more = true; while (more) { more = false; for (size_t n = 0; n < used_; ++n) { if (!buffers_[n]->active()) { // not active so we dont have to wait on it continue; } /* wait on just this buffer */ std::vector aioplist; aioplist.push_back(buffers_[n]->caioptr()); errno = 0; while (::aio_suspend(aioplist.data(), aioplist.size(), nullptr) < 0) { if (errno != EINTR) { throw FailedSystemCall("aio_suspend"); } } int e = ::aio_error(buffers_[n]->caioptr()); if (e == EINPROGRESS) { more = true; continue; } buffers_[n]->active(false); if (e == 0) { ssize_t len = ::aio_return(buffers_[n]->aioptr()); if (len != buffers_[n]->length()) { // TODO: retry when filesystems are full std::ostringstream os; os << "AIOHandle: only " << len << " bytes written instead of " << buffers_[n]->length(); throw WriteError(os.str()); } } else { throw FailedSystemCall("aio_return"); } } } if (fsync_) { // request all current operations to the synchronized I/O completion state struct aiocb aio; eckit::zero(aio); aio.aio_fildes = fd_; aio.aio_sigevent.sigev_notify = SIGEV_NONE; SYSCALL(::aio_fsync(O_SYNC, &aio)); more = true; while (more) { more = false; /* wait */ const struct aiocb* aiop = &aio; errno = 0; while (::aio_suspend(&aiop, 1, nullptr) < 0) { if (errno != EINTR) { throw FailedSystemCall("aio_suspend"); } } int e = ::aio_error(&aio); if (e == EINPROGRESS) { more = 1; } else if (e != 0) { throw FailedSystemCall("aio_error"); } } } } #else // NO eckit_HAVE_AIO struct AIOBuffer { AIOBuffer() = default; AIOBuffer(const AIOBuffer&) = delete; AIOBuffer& operator=(const AIOBuffer&) = delete; ~AIOBuffer() = default; }; AIOHandle::AIOHandle(const PathName& path, size_t count, size_t size, bool fsync) { NOTIMP; } long AIOHandle::write(const void* buffer, long length) { NOTIMP; } void AIOHandle::flush() { NOTIMP; } #endif //---------------------------------------------------------------------------------------------------------------------- AIOHandle::~AIOHandle() { for (size_t i = 0; i < count_; i++) { delete buffers_[i]; } } Length AIOHandle::openForRead() { NOTIMP; } void AIOHandle::openForWrite(const Length&) { used_ = 0; SYSCALL2(fd_ = ::open(path_.localPath(), O_WRONLY | O_CREAT | O_TRUNC, 0777), path_); pos_ = 0; } void AIOHandle::openForAppend(const Length&) { used_ = 0; SYSCALL2(fd_ = ::open(path_.localPath(), O_WRONLY | O_CREAT | O_APPEND, 0777), path_); SYSCALL2(pos_ = ::lseek(fd_, 0, SEEK_CUR), path_); } long AIOHandle::read(void*, long) { NOTIMP; } void AIOHandle::close() { if (fd_ != -1) { flush(); // this waits for the async requests to finish SYSCALL(::close(fd_)); fd_ = -1; } } void AIOHandle::rewind() { NOTIMP; } void AIOHandle::print(std::ostream& s) const { s << "AIOHandle[" << path_ << ']'; } Length AIOHandle::size() { Stat::Struct info; SYSCALL(Stat::fstat(fd_, &info)); return info.st_size; } Length AIOHandle::estimate() { Stat::Struct info; SYSCALL(Stat::fstat(fd_, &info)); return info.st_size; } Offset AIOHandle::position() { return pos_; } std::string AIOHandle::title() const { return std::string("AIO[") + PathName::shorten(path_) + "]"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/MultiHandle.cc0000664000175000017500000002713515161702250020024 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/Resource.h" #include "eckit/io/MultiHandle.h" #include "eckit/log/Timer.h" #include "eckit/runtime/Metrics.h" #include "eckit/types/Types.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec MultiHandle::classSpec_ = { &DataHandle::classSpec(), "MultiHandle", }; Reanimator MultiHandle::reanimator_; MultiHandle::MultiHandle() : current_(datahandles_.end()), read_(false) {} MultiHandle::MultiHandle(const std::vector& v) : datahandles_(v), current_(datahandles_.end()), read_(false) {} MultiHandle::MultiHandle(Stream& s) : DataHandle(s), read_(false) { unsigned long size; s >> size; datahandles_.reserve(size); for (size_t i = 0; i < size; i++) { DataHandle* dh = Reanimator::reanimate(s); ASSERT(dh); datahandles_.push_back(dh); } s >> length_; current_ = datahandles_.end(); } void MultiHandle::encode(Stream& s) const { DataHandle::encode(s); s << datahandles_.size(); for (size_t i = 0; i < datahandles_.size(); i++) { s << *(datahandles_[i]); } s << length_; } MultiHandle::~MultiHandle() { for (size_t i = 0; i < datahandles_.size(); i++) { delete datahandles_[i]; } } void MultiHandle::operator+=(DataHandle* dh) { ASSERT(dh != nullptr); // Try to merge with self if (merge(dh)) { delete dh; return; } // Try to merge with last if (datahandles_.size() > 0) { if (datahandles_.back()->merge(dh)) { delete dh; return; } } // Add to end datahandles_.push_back(dh); } void MultiHandle::operator+=(const Length& length) { length_.push_back(length); } Length MultiHandle::openForRead() { read_ = true; current_ = datahandles_.begin(); openCurrent(); // compress(); return estimate(); } void MultiHandle::openForWrite(const Length& length) { ASSERT(length == std::accumulate(length_.begin(), length_.end(), Length(0))); ASSERT(datahandles_.size() == length_.size()); read_ = false; Log::info() << "MultiHandle::openForWrite " << length << std::endl; Log::info() << "MultiHandle::openForWrite " << datahandles_.size() << std::endl; Log::info() << "MultiHandle::openForWrite " << length_.size() << std::endl; current_ = datahandles_.begin(); curlen_ = length_.begin(); openCurrent(); written_ = 0; Log::info() << "MultiHandle::openForWrite " << length_.size() << std::endl; if (current_ != datahandles_.end()) { Log::info() << "MultiHandle::openForWrite " << (*curlen_) << std::endl; } else { Log::warning() << "MultiHandle::openForWrite is empty" << std::endl; } } void MultiHandle::openForAppend(const Length&) { NOTIMP; } void MultiHandle::openCurrent() { if (current_ != datahandles_.end()) { if (read_) { Log::debug() << *(*current_) << std::endl; Log::debug() << "Multi handle: open " << (*current_)->openForRead() << std::endl; } else { (*current_)->openForWrite(*curlen_); } } } long MultiHandle::read1(char* buffer, long length) { if (current_ == datahandles_.end()) { return 0; } long n = (*current_)->read(buffer, length); if (n < length) { (*current_)->close(); current_++; openCurrent(); if (n <= 0) { return read1(buffer, length); } } return n; } long MultiHandle::read(void* buffer, long length) { char* p = static_cast(buffer); long n = 0; long total = 0; while (length > 0 && (n = read1(p, length)) > 0) { length -= n; total += n; p += n; } Log::debug() << "MultiHandle::read " << (total > 0 ? total : n) << std::endl; return total > 0 ? total : n; } long MultiHandle::write(const void* buffer, long length) { Length len = std::min(Length(*curlen_ - written_), Length(length)); long l = static_cast(len); ASSERT(len == Length(l)); ASSERT((*current_)); long n = (*current_)->write(buffer, l); Log::debug() << "MultiHandle::write " << *(*current_) << " " << length << ' ' << *curlen_ << ' ' << len << ' ' << written_ << std::endl; if (n <= 0) { return n; } written_ += n; if (written_ == (*curlen_)) { (*current_)->close(); current_++; curlen_++; openCurrent(); written_ = 0; // Some more ? if (length > l) { if (current_ == datahandles_.end()) { Log::debug() << length << " " << l << std::endl; } ASSERT(current_ != datahandles_.end()); const void* pl = static_cast(buffer) + l; long m = write(pl, length - l); return (m > 0 ? n + m : n); } } return n; } void MultiHandle::close() { if (current_ != datahandles_.end()) { (*current_)->close(); } current_ = datahandles_.end(); } void MultiHandle::flush() { for (size_t i = 0; i < datahandles_.size(); ++i) { datahandles_[i]->flush(); } } void MultiHandle::rewind() { ASSERT(read_); if (current_ != datahandles_.end()) { (*current_)->close(); } current_ = datahandles_.begin(); openCurrent(); } void MultiHandle::print(std::ostream& s) const { if (format(s) == Log::compactFormat) { s << "MultiHandle"; } else { s << "MultiHandle["; for (size_t i = 0; i < datahandles_.size(); i++) { if (i != 0) { s << ",("; } datahandles_[i]->print(s); s << ")"; } s << ']'; } } bool MultiHandle::merge(DataHandle* other) { if (other->isEmpty()) { return true; } // Poor man's RTTI, // Does not support inheritance if (!sameClass(*other)) { return false; } // An other MultiHandle // We should be safe to cast now.... MultiHandle* handle = dynamic_cast(other); // Merge in all datahandles for (size_t i = 0; i < handle->datahandles_.size(); i++) { (*this) += handle->datahandles_[i]; } handle->datahandles_.clear(); return true; } Length MultiHandle::size() { Length total = 0; for (size_t i = 0; i < datahandles_.size(); i++) { total += datahandles_[i]->size(); } return total; } Length MultiHandle::estimate() { Length total = 0; for (size_t i = 0; i < datahandles_.size(); i++) { total += datahandles_[i]->estimate(); } return total; } DataHandle* MultiHandle::clone() const { MultiHandle* mh = new MultiHandle(); for (size_t i = 0; i < datahandles_.size(); i++) { (*mh) += datahandles_[i]->clone(); } return mh; } bool MultiHandle::canSeek() const { for (size_t i = 0; i < datahandles_.size(); i++) { if (!datahandles_[i]->canSeek()) { return false; } } return true; } Offset MultiHandle::position() { long long accumulated = 0; for (HandleList::iterator it = datahandles_.begin(); it != current_ && it != datahandles_.end(); ++it) { accumulated += (*it)->size(); } return accumulated + (current_ == datahandles_.end() ? Offset(0) : (*current_)->position()); } Offset MultiHandle::seek(const Offset& offset) { ASSERT(read_); /// seek only allowed on read mode if (current_ != datahandles_.end()) { (*current_)->close(); } const long long seekto = offset; long long accumulated = 0; for (current_ = datahandles_.begin(); current_ != datahandles_.end(); ++current_) { long long size = (*current_)->size(); if (accumulated <= seekto && seekto < accumulated + size) { openCurrent(); (*current_)->seek(seekto - accumulated); return offset; } accumulated += size; } // check if we went beyond EOF which is POSIX compliant, but we ASSERT so we find possible bugs Offset beyond = seekto - accumulated; ASSERT(not beyond); return offset; } void MultiHandle::restartReadFrom(const Offset& offset) { Log::warning() << *this << " restart read from " << offset << std::endl; ASSERT(read_); if (current_ != datahandles_.end()) { (*current_)->close(); } long long from = offset; long long accumulated = 0; for (current_ = datahandles_.begin(); current_ != datahandles_.end(); ++current_) { long long e = (*current_)->estimate(); if (from >= accumulated && from < accumulated + e) { Log::warning() << *this << " restart read from " << from << ", current=" << (current_ - datahandles_.begin()) << std::endl; openCurrent(); (*current_)->restartReadFrom(from - accumulated); return; } accumulated += e; } // check if we went beyond EOF which is POSIX compliant, but we ASSERT so we find possible bugs Offset beyond = from - accumulated; ASSERT(not beyond); } void MultiHandle::toRemote(Stream& s) const { s.startObject(); s << className(); DataHandle::encode(s); s << datahandles_.size(); for (size_t i = 0; i < datahandles_.size(); i++) { datahandles_[i]->toRemote(s); } s << length_; s.endObject(); } void MultiHandle::toLocal(Stream& s) const { s.startObject(); s << className(); DataHandle::encode(s); s << datahandles_.size(); for (size_t i = 0; i < datahandles_.size(); i++) { datahandles_[i]->toLocal(s); } s << length_; s.endObject(); } DataHandle* MultiHandle::toLocal() { for (size_t i = 0; i < datahandles_.size(); i++) { DataHandle* loc = datahandles_[i]->toLocal(); if (loc != datahandles_[i]) { delete datahandles_[i]; datahandles_[i] = loc; } } return this; } void MultiHandle::selectMover(MoverTransferSelection& c, bool read) const { for (size_t i = 0; i < datahandles_.size(); i++) { datahandles_[i]->selectMover(c, read); } } bool MultiHandle::moveable() const { for (size_t i = 0; i < datahandles_.size(); i++) { if (!datahandles_[i]->moveable()) { return false; } } return datahandles_.size() > 0; } std::string MultiHandle::title() const { std::ostringstream os; os << "["; if (datahandles_.size() > 0) { os << datahandles_[0]->title(); } if (datahandles_.size() > 1) { os << ",...{" << datahandles_.size() << "}"; } os << "]"; return os.str(); } void MultiHandle::collectMetrics(const std::string& what) const { if (datahandles_.size() == 1) { return datahandles_[0]->collectMetrics(what); } std::map v; for (size_t i = 0; i < datahandles_.size(); i++) { v[datahandles_[i]->metricsTag()] += datahandles_[i]->estimate(); } Metrics::set(what, v); } bool MultiHandle::compress(bool) { return false; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/BitIO.cc0000664000175000017500000001033715161702250016560 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/BitIO.h" #include "eckit/io/DataHandle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- BitIO::BitIO(DataHandle& handle, bool padded) : handle_(handle), buffer_(0), used_(0), bits_(0), write_(false), eof_(false), padded_(padded), opened_(false) {} BitIO::~BitIO() { if (write_) { flush(); } if (opened_) { handle_.close(); } } static unsigned char masks[] = { 0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF, }; void BitIO::write(size_t code, size_t nbits) { const size_t BITS = sizeof(buffer_) * 8; write_ = true; if (!opened_) { handle_.openForWrite(0); opened_ = true; } while (nbits) { if (used_ == BITS) { flush(); ASSERT(used_ == 0); ASSERT(buffer_ == 0); } size_t s = std::min(std::min(nbits, BITS - used_), size_t(8)); buffer_ <<= s; buffer_ |= (code >> (nbits - s)) & masks[s]; // code <<= s; used_ += s; nbits -= s; bits_ += s; } } void BitIO::flush() { // const size_t BITS = sizeof(buffer_) * 8; // buffer_ <<= (BITS - used_); // std::cout << "flush " << used_ // << " " << std::bitset(buffer_) << std::endl; // std::cout << "bits " << bitCount() << std::endl; size_t nbits = used_; // ccc-cccc.cccc while (nbits) { size_t s = std::min(nbits, size_t(8)); unsigned char c = (buffer_ >> (nbits - s)) & masks[s]; c <<= (8 - s); // std::cout << "write " << std::bitset<8>(c); // if(::isprint(c)) { // std::cout << " " << char(c); // } // std::cout << ' ' << s < 0) { padded_ = false; return result << (asked - count); } if (EOF_MARKER) { return EOF_MARKER; } std::ostringstream oss; oss << "Failed to read from " << handle_; throw eckit::ReadError(oss.str()); } buffer_ <<= (BITS - used_); } size_t s = std::min(std::min(nbits, used_), size_t(8)); result <<= s; result |= (buffer_ >> (BITS - s)) & masks[s]; buffer_ <<= s; used_ -= s; nbits -= s; count += s; } return result; } size_t BitIO::bitCount() const { return bits_; } size_t BitIO::byteCount() const { return (bits_ + 7) / 8; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/StdPipe.cc0000664000175000017500000000201115161702250017150 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/StdPipe.h" namespace eckit { StdPipe::StdPipe(const std::string& name, const std::string& mode) : file_{nullptr} { file_ = ::popen(name.c_str(), mode.c_str()); if (file_ == nullptr) { throw CantOpenFile(name); } } StdPipe::~StdPipe() { ASSERT_MSG(!isOpen(), "StdPipe hasn't been closed before destruction"); } void StdPipe::close() noexcept(false) { if (isOpen()) { if (::pclose(file_) == -1) { throw FailedSystemCall("pclose"); } } file_ = nullptr; } } // namespace eckit eckit-2.0.7/src/eckit/io/Length.cc0000664000175000017500000000240515161702250017030 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/io/Length.h" #include "eckit/persist/DumpLoad.h" #include "eckit/serialisation/Stream.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- std::ostream& operator<<(std::ostream& s, const Length& x) { return s << x.value_; } Stream& operator<<(Stream& s, const Length& x) { return s << x.value_; } Stream& operator>>(Stream& s, Length& x) { return s >> x.value_; } void Length::dump(DumpLoad& a) const { a.dump(value_); } void Length::load(DumpLoad& a) { a.load(value_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/PooledFile.h0000664000175000017500000000302715161702250017474 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 2019 #ifndef eckit_io_PooledFile_h #define eckit_io_PooledFile_h #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" namespace eckit { class PoolFileEntry; class PooledFile { public: PooledFile(const PathName& name); PooledFile(const PooledFile&) = delete; PooledFile& operator=(const PooledFile&) = delete; PooledFile(PooledFile&&) = delete; PooledFile& operator=(PooledFile&&) = delete; /// @pre must have been closed ~PooledFile(); void open(); /// @throws on fclose failure void close() noexcept(false); long read(void*, long); off_t seek(off_t offset); off_t seekEnd(); off_t rewind(); int fileno() const; // for testing size_t nbOpens() const; size_t nbReads() const; size_t nbSeeks() const; private: PathName name_; PoolFileEntry* entry_; }; class PooledFileError : public FileError { public: PooledFileError(const std::string& file, const std::string& msg, const CodeLocation& loc); }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/CommandStream.h0000664000175000017500000000242115161702250020201 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File CommandStream.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_CommandStream_h #define eckit_CommandStream_h #include "eckit/serialisation/Stream.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class CommandStream : public Stream { public: // -- Contructors CommandStream(const std::string& name, const char* mode); // -- Destructor ~CommandStream(); // -- Overridden methods // From Stream long read(void*, long) override; long write(const void*, long) override; private: // -- Members FILE* file_; // -- Overridden methods // From Stream std::string name() const override; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/PartFileHandle.cc0000664000175000017500000001653315161702250020440 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/io/cluster/NodeInfo.h" #include "eckit/log/Log.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/PartFileHandle.h" #include "eckit/io/MoverTransferSelection.h" #include "eckit/io/PooledHandle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec PartFileHandle::classSpec_ = { &DataHandle::classSpec(), "PartFileHandle", }; Reanimator PartFileHandle::reanimator_; void PartFileHandle::print(std::ostream& s) const { if (format(s) == Log::compactFormat) { s << "PartFileHandle"; } else { s << "PartFileHandle[path=" << path_ << ",offset=" << offset_ << ",length=" << length_ << ']'; } } void PartFileHandle::encode(Stream& s) const { DataHandle::encode(s); s << path_; s << offset_; s << length_; } PartFileHandle::PartFileHandle(Stream& s) : DataHandle(s), pos_(0), index_(0) { s >> path_; s >> offset_; s >> length_; ASSERT(offset_.size() == length_.size()); } PartFileHandle::PartFileHandle(const PathName& name, const OffsetList& offset, const LengthList& length) : path_(name), handle_(), pos_(0), index_(0), offset_(offset), length_(length) { // Log::info() << "PartFileHandle::PartFileHandle " << name << std::endl; ASSERT(offset_.size() == length_.size()); compress(false); } PartFileHandle::PartFileHandle(const PathName& name, const Offset& offset, const Length& length) : path_(name), handle_(), pos_(0), index_(0), offset_(1, offset), length_(1, length) {} DataHandle* PartFileHandle::clone() const { return new PartFileHandle(path_, offset_, length_); } bool PartFileHandle::compress(bool sorted) { if (sorted) { eckit::sort(offset_, length_); } return eckit::compress(offset_, length_); } PartFileHandle::~PartFileHandle() {} Length PartFileHandle::openForRead() { if (!handle_) { // The handle may already exists if a restartReadFrom() // is requested handle_.reset(new PooledHandle(path_)); } handle_->openForRead(); rewind(); return estimate(); } void PartFileHandle::openForWrite(const Length&) { NOTIMP; } void PartFileHandle::openForAppend(const Length&) { NOTIMP; } long PartFileHandle::read1(char* buffer, long length) { ASSERT(handle_); // skip empty entries if any while (index_ < offset_.size() && length_[index_] == Length(0)) { index_++; } if (index_ == offset_.size()) { return 0; } Length ll = (long long)offset_[index_] + Length(pos_); off_t pos = ll; handle_->seek(pos); ll = length_[index_] - Length(pos_); Length lsize = std::min(Length(length), ll); long size = lsize; ASSERT(Length(size) == lsize); long n = handle_->read(buffer, size); if (n != size) { std::ostringstream s; s << path_ << ": cannot read " << size << ", got only " << n; throw ReadError(s.str()); } pos_ += n; if (pos_ >= length_[index_]) { index_++; pos_ = 0; } return n; } long PartFileHandle::read(void* buffer, long length) { char* p = (char*)buffer; long n = 0; long total = 0; while (length > 0 && (n = read1(p, length)) > 0) { length -= n; total += n; p += n; } return total > 0 ? total : n; } long PartFileHandle::write(const void*, long) { NOTIMP; } void PartFileHandle::close() { if (handle_) { handle_->close(); // Don't delete the handle here so the PooledHandle entry continues // to live, so the underlying file is not closed, which give a chance to PooledHandle // to do its work, for example of that PartFileHandle is itself part of // a multihandle that points many time to the same path // handle_.reset(); } } void PartFileHandle::rewind() { pos_ = 0; index_ = 0; } void PartFileHandle::restartReadFrom(const Offset& from) { Log::warning() << *this << " restart read from " << from << std::endl; rewind(); long long len = from; long long pos = 0; for (index_ = 0; index_ < length_.size(); index_++) { long long e = length_[index_]; if (len >= pos && len < pos + e) { Log::warning() << *this << " restart read from " << from << ", index=" << index_ << ", pos=" << pos_ << std::endl; pos_ = len - pos; return; } pos += e; } ASSERT(from == Offset(0) && estimate() == Length(0)); } Offset PartFileHandle::position() { long long position = 0; for (Ordinal i = 0; i < index_; i++) { position += length_[i]; } return position + Length(pos_); } Offset PartFileHandle::seek(const Offset& offset) { rewind(); const long long seekto = offset; long long accumulated = 0; for (index_ = 0; index_ < length_.size(); index_++) { long long len = length_[index_]; if (accumulated <= seekto && seekto < accumulated + len) { pos_ = seekto - accumulated; return offset; } accumulated += len; } long long beyond = seekto - accumulated; // check if seek went beyond EOF which is POSIX compliant, but we ASSERT so we find possible bugs ASSERT(not beyond); pos_ = 0; return seekto; } bool PartFileHandle::canSeek() const { return true; } bool PartFileHandle::merge(DataHandle* other) { if (other->isEmpty()) { return true; } // Poor man's RTTI, // Does not support inheritance if (!sameClass(*other)) { return false; } // We should be safe to cast now.... PartFileHandle* handle = dynamic_cast(other); if (path_ != handle->path_) { return false; } ASSERT(handle->offset_.size() == handle->length_.size()); offset_.reserve(offset_.size() + handle->offset_.size()); length_.reserve(length_.size() + handle->length_.size()); for (Ordinal i = 0; i < handle->offset_.size(); ++i) { offset_.push_back(handle->offset_[i]); length_.push_back(handle->length_[i]); } compress(false); return true; } Length PartFileHandle::size() { return std::accumulate(length_.begin(), length_.end(), Length(0)); } Length PartFileHandle::estimate() { return std::accumulate(length_.begin(), length_.end(), Length(0)); } void PartFileHandle::selectMover(MoverTransferSelection& c, bool read) const { if (read) { c.updateCost(path_.node(), const_cast(this)->estimate()); } } std::string PartFileHandle::title() const { std::ostringstream os; os << PathName::shorten(path_) << " (" << length_.size() << ")"; return os.str(); } std::string PartFileHandle::metricsTag() const { return PathName::metricsTag(path_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/MoverTransferSelection.h0000664000175000017500000000441715161702250022121 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File MoverTransferSelection.h // Baudouin Raoult - (c) ECMWF Jun 23 #ifndef eckit_MoverTransferSelection_h #define eckit_MoverTransferSelection_h #include #include #include #include "eckit/io/Length.h" //----------------------------------------------------------------------------- namespace eckit { class NodeInfo; //----------------------------------------------------------------------------- class MoverTransferSelection { public: MoverTransferSelection(); ~MoverTransferSelection(); // -- Methods void updateCost(const NodeInfo&, const Length& length); void updateCost(const std::string&, const Length& length); void requiredMoverAttributes(const std::set& attrs); void preferredMover(const NodeInfo&); void preferredMover(const std::string&); NodeInfo selectedMover(); // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods // void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed MoverTransferSelection(const MoverTransferSelection&); MoverTransferSelection& operator=(const MoverTransferSelection&); // -- Members std::set moverAttributes_; std::map cost_; std::string preferredMover_; // -- Methods void selectedMover(NodeInfo&, bool&); // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends // friend std::ostream& operator<<(std::ostream& s,const MoverTransfer& p) // { p.print(s); return s; } }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/FDataSync.h0000664000175000017500000000125415161702250017266 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @date Dec 2018 #ifndef eckit_io_FDataSync_h #define eckit_io_FDataSync_h namespace eckit { /// An fsync that retries when interrupted by system int fsync(int fd); /// A platform independent fdatasync int fdatasync(int fd); } // namespace eckit #endif eckit-2.0.7/src/eckit/io/MMappedFileHandle.cc0000664000175000017500000001022315161702250021043 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // #include // #include #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/MMappedFileHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/memory/MMap.h" #include "eckit/os/Stat.h" #include "eckit/utils/MD5.h" namespace eckit { ClassSpec MMappedFileHandle::classSpec_ = { &DataHandle::classSpec(), "MMappedFileHandle", }; Reanimator MMappedFileHandle::reanimator_; void MMappedFileHandle::print(std::ostream& s) const { s << "MMappedFileHandle[file=" << path_ << ']'; } void MMappedFileHandle::encode(Stream& s) const { DataHandle::encode(s); s << path_; } MMappedFileHandle::MMappedFileHandle(Stream& s) : DataHandle(s), mmap_{nullptr}, fd_(-1) { s >> path_; } MMappedFileHandle::MMappedFileHandle(const std::string& path) : path_(path), mmap_{nullptr}, fd_(-1) {} MMappedFileHandle::~MMappedFileHandle() {} Length MMappedFileHandle::openForRead() { ASSERT(!handle_.get()); Stat::Struct info; SYSCALL(Stat::stat(path_.c_str(), &info)); length_ = info.st_size; SYSCALL2(fd_ = ::open(path_.c_str(), O_RDONLY), path_); mmap_ = MMap::mmap(nullptr, length_, PROT_READ, MAP_SHARED, fd_, 0); if (mmap_ == MAP_FAILED) { Log::error() << "MMappedFileHandle path=" << path_ << " size=" << length_ << " fails to mmap(0,length,PROT_READ,MAP_SHARED,fd_,0)" << Log::syserr << std::endl; throw FailedSystemCall("mmap", Here()); } handle_.reset(new MemoryHandle(mmap_, length_)); return handle_->openForRead(); } void MMappedFileHandle::openForWrite(const Length& length) { NOTIMP; } void MMappedFileHandle::openForAppend(const Length&) { NOTIMP; } long MMappedFileHandle::read(void* buffer, long length) { ASSERT(handle_.get()); return handle_->read(buffer, length); } long MMappedFileHandle::write(const void* buffer, long length) { ASSERT(handle_.get()); return handle_->write(buffer, length); } void MMappedFileHandle::flush() { ASSERT(handle_.get()); return handle_->flush(); } void MMappedFileHandle::close() { if (handle_.get()) { handle_->close(); handle_.reset(nullptr); } if (mmap_) { SYSCALL2(MMap::munmap(mmap_, length_), path_); mmap_ = nullptr; } if (fd_ >= 0) { SYSCALL2(::close(fd_), path_); fd_ = -1; } } void MMappedFileHandle::rewind() { ASSERT(handle_.get()); return handle_->rewind(); } Length MMappedFileHandle::size() { ASSERT(handle_.get()); return handle_->size(); } Length MMappedFileHandle::estimate() { ASSERT(handle_.get()); return handle_->estimate(); } bool MMappedFileHandle::isEmpty() const { ASSERT(handle_.get()); return handle_->isEmpty(); } Offset MMappedFileHandle::position() { ASSERT(handle_.get()); return handle_->position(); } void MMappedFileHandle::restartReadFrom(const Offset& from) { ASSERT(handle_.get()); handle_->restartReadFrom(from); } void MMappedFileHandle::restartWriteFrom(const Offset& from) { ASSERT(handle_.get()); handle_->restartWriteFrom(from); } Offset MMappedFileHandle::seek(const Offset& from) { ASSERT(handle_.get()); return handle_->seek(from); } void MMappedFileHandle::skip(const Length& n) { ASSERT(handle_.get()); handle_->skip(n); } std::string MMappedFileHandle::title() const { return "mmap(" + PathName::shorten(path_) + ")"; } std::string MMappedFileHandle::metricsTag() const { return "mmap(" + PathName::metricsTag(path_) + ")"; } DataHandle* MMappedFileHandle::clone() const { return new MMappedFileHandle(path_); } void MMappedFileHandle::hash(MD5& md5) const { md5 << "MMappedFileHandle"; md5 << path_; } } // namespace eckit eckit-2.0.7/src/eckit/io/FilePool.h0000664000175000017500000000573415161702250017172 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @date Dec 2015 #ifndef eckit_io_FilePool_h #define eckit_io_FilePool_h #include "eckit/container/CacheLRU.h" #include "eckit/thread/MutexCond.h" namespace eckit { class DataHandle; class PathName; } // namespace eckit namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// Pool of a maximum number of open files. /// Open files manipulated via DataHandle interface. /// Handles are kept open as long as possible following LRU algorithm /// Handles must be checked-out before usage, should be checked back in once writing finished. /// No limit to number of checked out handles. /// Handles are closed when purged from pool due to LRU or when pool is destroyed. /// /// @note this class is thread-safe /// class FilePool { public: FilePool(size_t capacity); FilePool(const FilePool&) = delete; FilePool& operator=(const FilePool&) = delete; FilePool(FilePool&&) = delete; FilePool& operator=(FilePool&&) = delete; ~FilePool(); /// Checkout a DataHandle for use /// Ownership is passed from FilePool to the client /// @post DataHandle is marked in use and should not be closed by client /// @invariant If handle is in use, it assumes is from another thread and it will wait() for a checkin() DataHandle* checkout(const PathName& path); /// Return a DataHandle after use /// Ownership is passed back from the client to FilePool /// @post DataHandle is marked out of use and may now be closed by FilePool void checkin(DataHandle* handle); /// Remove a DataHandle from the pool /// @invariant If handle is in use, it assumes is from another thread and it will wait() for a checkin() bool remove(const PathName& path); /// Current size of pool size_t size() const; /// Resize pool capacity void capacity(size_t size); /// Returns max size of pool size_t capacity() const; /// Returns number of checked-out DataHandles size_t usage() const; /// Lists the contents of the Pool both in use and cached DataHandle's void print(std::ostream& os) const; friend std::ostream& operator<<(std::ostream& s, const FilePool& p) { p.print(s); return s; } private: std::map inUse_; eckit::CacheLRU cache_; mutable eckit::MutexCond cond_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/MoverTransfer.cc0000664000175000017500000000574715161702250020420 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/MoverTransfer.h" #include "eckit/io/MoverTransferSelection.h" #include "eckit/log/Bytes.h" #include "eckit/log/Progress.h" #include "eckit/net/Connector.h" #include "eckit/runtime/Metrics.h" #include "eckit/runtime/Monitor.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- MoverTransfer::MoverTransfer(TransferWatcher& watcher) : watcher_(watcher) {} MoverTransfer::~MoverTransfer() {} Length MoverTransfer::transfer(DataHandle& from, DataHandle& to) { bool send_costs = true; if (!from.moveable()) { throw SeriousBug(from.title() + " is not moveable"); } if (!to.moveable()) { throw SeriousBug(to.title() + " is not moveable"); } // Using node-specific info, determine beneficial nodes to use MoverTransferSelection cost; from.selectMover(cost, true); to.selectMover(cost, false); Log::info() << "MoverTransfer::transfer(" << from << "," << to << ")" << std::endl; net::Connector& c(net::Connector::service(cost.selectedMover())); AutoLock lock(c); // This will close the connector on unlock c.autoclose(true); Log::message() << c.node() << std::endl; Stream& s = c; s << bool(false); // New batch // NodeInfo::sendLogin(s); // NodeInfo remote = NodeInfo::acceptLogin(s); Log::status() << "Sending input handle " << from.title() << std::endl; from.toRemote(s); Log::status() << "Sending output handle " << to.title() << std::endl; to.toRemote(s); Length estimate = from.estimate(); AutoState state('M'); if (estimate) { Log::status() << Bytes(estimate) << " "; } Log::status() << from.title() << " => " << to.title() << std::endl; Progress progress("mover", 0, estimate); watcher_.watch(nullptr, 0); unsigned long long total = 0; bool more; s >> more; while (more) { long pos; std::string msg; s >> pos; if (pos >= 0) { s >> msg; total += pos; progress(total); watcher_.watch(nullptr, total); } else if (pos == -1) { watcher_.fromHandleOpened(); } else if (pos == -2) { watcher_.toHandleOpened(); } s >> more; } unsigned long long len; s >> len; Metrics::receive(s); Log::message() << " " << std::endl; return len; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/CommandStream.cc0000664000175000017500000000307215161702250020342 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/CommandStream.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- CommandStream::CommandStream(const std::string& name, const char* mode) : file_(popen(name.c_str(), mode)) { Log::info() << "CommandStream (" << name << ")" << std::endl; if (file_ == nullptr) { throw CantOpenFile(name); } } CommandStream::~CommandStream() { // Somethings wrong here, throw in a dtor ?? // if(pclose(file_)) // throw WriteError("CommandStream::~CommandStream()"); } long CommandStream::read(void* buf, long length) { return fread(buf, 1, length, file_); } long CommandStream::write(const void* buf, long length) { return fwrite(buf, 1, length, file_); } std::string CommandStream::name() const { return "CommandStream"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/StdioBuf.h0000664000175000017500000000245415161702250017174 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File StdioBuf.h // Baudouin Raoult - ECMWF Mar 97 #ifndef eckit_StdioBuf_h #define eckit_StdioBuf_h #include #include "eckit/eckit.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class StdioBuf : public std::streambuf { public: // -- Contructors StdioBuf(FILE*); // -- Destructor ~StdioBuf(); private: // No copy allowed StdioBuf(const StdioBuf&); StdioBuf& operator=(const StdioBuf&); // -- Members char in_[1]; char out_[80]; FILE* file_; // -- Overridden methods // From streambuf virtual int overflow(int c); virtual int underflow(); virtual int sync(); // virtual int uflow(); }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/FDataSync.cc0000664000175000017500000000311415161702250017421 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "FDataSync.h" #include #include #include "eckit/eckit.h" #include "eckit/os/Stat.h" namespace eckit { int fsync(int fd) { Stat::Struct info; if (Stat::fstat(fd, &info) == 0 && !S_ISREG(info.st_mode) && !S_ISLNK(info.st_mode)) { return 0; } int ret = ::fsync(fd); while (ret < 0 && errno == EINTR) { ret = ::fsync(fd); } return ret; } int fdatasync(int fd) { int ret = 0; #if eckit_HAVE_FDATASYNC // usually available on Linux, but not Darwin (macosx) and xBSD // syncs all the data but avoids some of the metadata e.g. mtime ret = ::fdatasync(fd); #elif eckit_HAVE_F_FULLSYNC // usually available on Darwin (macosx) and xBSD // provides stronger guarantees than fsync that data fully committed to persistent storage ret = ::fcntl(fd, F_FULLFSYNC); while (ret == -1 && errno == EINTR) { ret = ::fcntl(fd, F_FULLFSYNC); } #elif eckit_HAVE_FSYNC // last resort, but note this is slower than ::fdatasync and less strong as F_FULLSYNC ret = eckit::fsync(fd); #else #error "Operating system does not support fdatasync, F_FULLSYNC or fsync" #endif return ret; } } // namespace eckit eckit-2.0.7/src/eckit/io/EmptyHandle.cc0000664000175000017500000000172015161702250020020 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/EmptyHandle.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec EmptyHandle::classSpec_ = { &DataHandle::classSpec(), "EmptyHandle", }; Reanimator EmptyHandle::reanimator_; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/TeeHandle.cc0000664000175000017500000001046415161702250017444 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/TeeHandle.h" #include "eckit/exception/Exceptions.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec TeeHandle::classSpec_ = { &DataHandle::classSpec(), "TeeHandle", }; Reanimator TeeHandle::reanimator_; TeeHandle::TeeHandle() {} TeeHandle::TeeHandle(const std::vector& v) : datahandles_(v) {} TeeHandle::TeeHandle(DataHandle* a, DataHandle* b) { datahandles_.push_back(a); datahandles_.push_back(b); } TeeHandle::TeeHandle(Stream& s) : DataHandle(s) { unsigned long size; s >> size; datahandles_.reserve(size); for (size_t i = 0; i < size; i++) { DataHandle* dh = Reanimator::reanimate(s); ASSERT(dh); datahandles_.push_back(dh); } } void TeeHandle::encode(Stream& s) const { DataHandle::encode(s); s << datahandles_.size(); for (size_t i = 0; i < datahandles_.size(); i++) { s << *(datahandles_[i]); } } TeeHandle::~TeeHandle() { for (size_t i = 0; i < datahandles_.size(); i++) { delete datahandles_[i]; } } void TeeHandle::operator+=(DataHandle* dh) { ASSERT(dh != nullptr); datahandles_.push_back(dh); } Length TeeHandle::openForRead() { NOTIMP; } void TeeHandle::openForWrite(const Length& length) { for (size_t i = 0; i < datahandles_.size(); i++) { datahandles_[i]->openForWrite(length); } } void TeeHandle::openForAppend(const Length&) { NOTIMP; } long TeeHandle::read(void*, long) { NOTIMP; } long TeeHandle::write(const void* buffer, long length) { long len = 0; for (size_t i = 0; i < datahandles_.size(); i++) { long l = datahandles_[i]->write(buffer, length); if (i) { ASSERT(len == l); } len = l; } return len; } void TeeHandle::close() { for (size_t i = 0; i < datahandles_.size(); i++) { datahandles_[i]->close(); } } void TeeHandle::flush() { for (size_t i = 0; i < datahandles_.size(); i++) { datahandles_[i]->flush(); } } void TeeHandle::rewind() { NOTIMP; } void TeeHandle::print(std::ostream& s) const { if (format(s) == Log::compactFormat) { s << "TeeHandle"; } else { s << "TeeHandle["; for (size_t i = 0; i < datahandles_.size(); i++) { if (i != 0) { s << ",("; } datahandles_[i]->print(s); s << ")"; } s << ']'; } } void TeeHandle::toRemote(Stream& s) const { s.startObject(); s << className(); DataHandle::encode(s); s << datahandles_.size(); for (size_t i = 0; i < datahandles_.size(); i++) { datahandles_[i]->toRemote(s); } s.endObject(); } void TeeHandle::toLocal(Stream& s) const { s.startObject(); s << className(); DataHandle::encode(s); s << datahandles_.size(); for (size_t i = 0; i < datahandles_.size(); i++) { datahandles_[i]->toLocal(s); } s.endObject(); } DataHandle* TeeHandle::toLocal() { for (size_t i = 0; i < datahandles_.size(); i++) { DataHandle* loc = datahandles_[i]->toLocal(); if (loc != datahandles_[i]) { delete datahandles_[i]; datahandles_[i] = loc; } } return this; } void TeeHandle::selectMover(MoverTransferSelection& c, bool read) const { for (size_t i = 0; i < datahandles_.size(); i++) { datahandles_[i]->selectMover(c, read); } } bool TeeHandle::moveable() const { for (const auto& dh : datahandles_) { if (!dh->moveable()) { return false; } } return true; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/SharedBuffer.cc0000664000175000017500000000216715161702250020154 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/SharedBuffer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- SharedBuffer::SharedBuffer(size_t size) : buffer_(new CountedBuffer(size)) { ASSERT(buffer_); buffer_->attach(); } SharedBuffer::SharedBuffer(CountedBuffer* b) { ASSERT(b); buffer_ = b; buffer_->attach(); } void SharedBuffer::print(std::ostream& os) const { os << "SharedBuffer(address=" << buffer_ << ",count=" << buffer_->count() << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/Select.cc0000664000175000017500000000650415161702250017032 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // Disable warnings: warning #550-D: variable "__d0" was set but never used [set_but_not_used] in FD_ZERO(&r); #if defined(__NVCOMPILER) #pragma diag_suppress 550 #endif #include #include #include "eckit/io/Select.h" #include "eckit/net/TCPSocket.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Select::Select() : last_(-1) { FD_ZERO(&files_); } Select::Select(net::TCPSocket& p) : last_(-1) { FD_ZERO(&files_); add(p); } Select::Select(int fd) : last_(-1) { FD_ZERO(&files_); add(fd); } Select::~Select() {} void Select::add(int fd) { ASSERT(fd >= 0 && fd < getdtablesize()); FD_SET(fd, &files_); if (fd > last_) { last_ = fd; } } void Select::add(net::TCPSocket& p) { add(p.socket()); } void Select::remove(int fd) { ASSERT(fd >= 0 && fd < getdtablesize()); FD_CLR(fd, &files_); } void Select::remove(net::TCPSocket& p) { remove(p.socket()); } bool Select::set(int fd) { ASSERT(fd >= 0 && fd < getdtablesize()); return FD_ISSET(fd, &set_); } bool Select::set(net::TCPSocket& p) { return set(p.socket()); } bool Select::ready(long sec) { int size = last_ + 1; ::timeval timeout; timeout.tv_sec = sec; timeout.tv_usec = 0; for (;;) { // First check with ioctl, as select is not always trustworthy bool some = false; FD_ZERO(&set_); for (int i = 0; i < size; i++) { if (FD_ISSET(i, &files_)) { int nbytes = 0; // cout << "ioctl(i,FIONREAD,&nbytes) " << i << " .... " << ioctl(i,FIONREAD,&nbytes) << " " << nbytes // << std::endl; nbytes = 0; // SYSCALL(ioctl(i,FIONREAD,&nbytes)); // // On Linux, a socket in acceoct() mode will return "Invalid argument" // so we simply ignor ethe error here.... if ((ioctl(i, FIONREAD, &nbytes) == 0) && (nbytes > 0)) { FD_SET(i, &set_); some = true; } } } if (some) { return true; } for (;;) { set_ = files_; fd_set excep = files_; switch (::select(size, &set_, nullptr, &excep, &timeout)) { case -1: if (errno != EINTR) { throw FailedSystemCall("select"); } break; case 0: return false; default: return true; } } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/StatsHandle.cc0000664000175000017500000001277115161702250020030 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit.h" #include "eckit/config/Resource.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/io/StatsHandle.h" #include "eckit/log/BigNum.h" #include "eckit/log/Bytes.h" #include "eckit/log/Seconds.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- StatsHandle::StatsHandle(DataHandle& handle) : HandleHolder(handle), reads_(0), seeks_(0), writes_(0), // positions_(0), bytesRead_(0), bytesWritten_(0), timer_(), readTime_(0), writeTime_(0), seekTime_(0) {} StatsHandle::StatsHandle(DataHandle* handle) : HandleHolder(handle), reads_(0), seeks_(0), writes_(0), // positions_(0), bytesRead_(0), bytesWritten_(0), timer_(), readTime_(0), writeTime_(0), seekTime_(0) {} StatsHandle::~StatsHandle() { std::cout << "StatsHandle for " << handle() << std::endl; std::cout << " Elapsed: " << eckit::Seconds(timer_.elapsed()) << std::endl; if (reads_) { std::cout << " No. of reads: " << eckit::BigNum(reads_) << std::endl; std::cout << " Bytes read: " << eckit::Bytes(bytesRead_) << std::endl; std::cout << " Average read: " << eckit::Bytes(bytesRead_ / reads_) << std::endl; std::cout << " Read time: " << eckit::Seconds(readTime_) << std::endl; std::cout << " Read rate: " << eckit::Bytes(bytesRead_, readTime_) << std::endl; } if (writes_) { std::cout << " No. of writes: " << eckit::BigNum(writes_) << std::endl; std::cout << " Bytes written: " << eckit::Bytes(bytesWritten_) << std::endl; std::cout << " Average write: " << eckit::Bytes(bytesWritten_ / writes_) << std::endl; std::cout << " Write time: " << eckit::Seconds(writeTime_) << std::endl; std::cout << " Write rate: " << eckit::Bytes(bytesWritten_, writeTime_) << std::endl; } if (seeks_) { std::cout << " No. of seeks: " << eckit::BigNum(seeks_) << std::endl; std::cout << " Seek time: " << eckit::Seconds(seekTime_) << std::endl; } } void StatsHandle::print(std::ostream& s) const { /* if(format(s) == Log::compactFormat) s << "StatsHandle"; else*/ s << "StatsHandle[handle=" << handle() << ']'; } Length StatsHandle::openForRead() { return handle().openForRead(); } void StatsHandle::openForWrite(const Length& l) { handle().openForWrite(l); } void StatsHandle::openForAppend(const Length& l) { handle().openForAppend(l); } long StatsHandle::read(void* data, long len) { double x = timer_.elapsed(); reads_++; bytesRead_ += len; long ret = handle().read(data, len); readTime_ += timer_.elapsed() - x; return ret; } long StatsHandle::write(const void* data, long len) { double x = timer_.elapsed(); writes_++; bytesWritten_ += len; long ret = handle().write(data, len); writeTime_ += timer_.elapsed() - x; return ret; } void StatsHandle::close() { handle().close(); } void StatsHandle::flush() { return handle().flush(); } Length StatsHandle::estimate() { return handle().estimate(); } Offset StatsHandle::position() { return handle().position(); } Offset StatsHandle::seek(const Offset& o) { double x = timer_.elapsed(); seeks_++; Offset ret = handle().seek(o); seekTime_ += timer_.elapsed() - x; return ret; } void StatsHandle::skip(const Length& n) { double x = timer_.elapsed(); seeks_++; handle().skip(n); seekTime_ += timer_.elapsed() - x; } void StatsHandle::rewind() { double x = timer_.elapsed(); seeks_++; handle().rewind(); seekTime_ += timer_.elapsed() - x; } void StatsHandle::restartReadFrom(const Offset& o) { return handle().restartReadFrom(o); } void StatsHandle::restartWriteFrom(const Offset& o) { return handle().restartWriteFrom(o); } DataHandle* StatsHandle::clone() const { NOTIMP; } std::string StatsHandle::name() const { return handle().name(); } bool StatsHandle::compress(bool b) { return handle().compress(b); } bool StatsHandle::merge(DataHandle*) { NOTIMP; } bool StatsHandle::isEmpty() const { return handle().isEmpty(); } bool StatsHandle::moveable() const { return false; } void StatsHandle::toLocal(Stream& s) const { NOTIMP; } DataHandle* StatsHandle::toLocal() { NOTIMP; } void StatsHandle::toRemote(Stream& s) const { NOTIMP; } void StatsHandle::selectMover(MoverTransferSelection&, bool) const { NOTIMP; } std::string StatsHandle::title() const { return handle().title(); } void StatsHandle::collectMetrics(const std::string& what) const { handle().collectMetrics(what); } Length StatsHandle::saveInto(DataHandle& other, TransferWatcher& watcher) { return handle().saveInto(other, watcher); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/HandleHolder.cc0000664000175000017500000000246715161702250020150 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit.h" #include "eckit/config/Resource.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/io/HandleHolder.h" #include "eckit/log/BigNum.h" #include "eckit/log/Bytes.h" #include "eckit/log/Seconds.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- HandleHolder::HandleHolder(DataHandle& handle) : handle_(&handle), owned_(false) {} HandleHolder::HandleHolder(DataHandle* handle) : handle_(handle), owned_(true) {} HandleHolder::~HandleHolder() { if (owned_) { delete handle_; } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/EasyCURL.cc0000664000175000017500000005430115161702250017200 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // Disable warnings: warning #550-D: variable "__d0" was set but never used [set_but_not_used] in FD_ZERO(&r); #if defined(__NVCOMPILER) #pragma diag_suppress 550 #endif #include "eckit/io/EasyCURL.h" #include #include #include #include "eckit/log/Log.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" #include "eckit/io/BufferedHandle.h" #include "eckit/io/CircularBuffer.h" #include "eckit/log/Bytes.h" #include "eckit/log/Timer.h" #include "eckit/parser/JSONParser.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Tokenizer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- namespace { struct http_code { int code_; const char* message_; bool retriable_; bool redirect_; }; std::vector http_codes = { { 100, "Continue", false, false, }, { 101, "Switching Protocols", false, false, }, { 102, "Processing", false, false, }, { 103, "Early Hints", false, false, }, { 200, "OK", false, false, }, { 201, "Created", false, false, }, { 202, "Accepted", false, false, }, { 203, "Non-Authoritative Information", false, false, }, { 204, "No Content", false, false, }, { 205, "Reset Content", false, false, }, { 206, "Partial Content", false, false, }, { 300, "Multiple Choices", false, false, }, { 301, "Moved Permanently", false, true, }, { 302, "Found", false, true, }, { 303, "See Other", false, true, }, { 304, "Not Modified", false, false, }, { 305, "Use Proxy", false, false, }, { 306, "Switch Proxy", false, false, }, { 307, "Temporary Redirect", false, true, }, { 308, "Permanent Redirect", false, true, }, { 400, "Bad Request", false, false, }, { 401, "Unauthorized", false, false, }, { 402, "Payment Required", false, false, }, { 403, "Forbidden", false, false, }, { 404, "Not Found", false, false, }, { 405, "Method Not Allowed", false, false, }, { 406, "Not Acceptable", false, false, }, { 407, "Proxy Authentication Required", false, false, }, { 408, "Request Timeout", false, false, }, { 409, "Conflict", false, false, }, { 410, "Gone", false, false, }, { 411, "Length Required", false, false, }, { 412, "Precondition Failed", false, false, }, { 413, "Request Entity Too Large", false, false, }, { 414, "URI Too Long", false, false, }, { 415, "Unsupported Media Type", false, false, }, { 416, "Range Not Satisfiable", false, false, }, { 417, "Expectation Failed", false, false, }, { 418, "I'm a teapot", false, false, }, { 421, "Misdirected Request", false, false, }, { 422, "Unprocessable Entity", false, false, }, { 423, "Locked", false, false, }, { 425, "Too Early", true, false, }, { 426, "Upgrade Required", false, false, }, { 428, "Precondition Required", false, false, }, { 429, "Too Many Requests", true, false, }, { 431, "Request Header Fields Too Large", false, false, }, { 451, "Unavailable For Legal Reasons", false, false, }, { 500, "Internal Server Error", true, false, }, { 501, "Not Implemented", false, false, }, { 502, "Bad Gateway", true, false, }, { 503, "Service Unavailable", true, false, }, { 504, "Gateway Timeout", true, false, }, { 505, "HTTP Version Not Supported", false, false, }, { 506, "Variant Also Negotiates", false, false, }, { 507, "Insufficient Storage", false, false, }, { 510, "Not Extended", false, false, }, { 511, "Network Authentication Required", false, false, }, }; } // namespace //---------------------------------------------------------------------------------------------------------------------- #define _(a) call(#a, a) static void call(const char* what, CURLcode code) { if (code != CURLE_OK) { std::ostringstream oss; oss << what << " failed: " << curl_easy_strerror(code); throw SeriousBug(oss.str()); } } static void call(const char* what, CURLMcode code) { if (code != CURLM_OK) { std::ostringstream oss; oss << what << " failed: " << curl_multi_strerror(code); throw SeriousBug(oss.str()); } } static pthread_once_t once = PTHREAD_ONCE_INIT; static CURLM* multi = nullptr; static void init() { _(curl_global_init(CURL_GLOBAL_DEFAULT)); multi = curl_multi_init(); } void EasyCURL::print(std::ostream& s) const { s << "EasyCURL[]"; } class CURLHandle : public eckit::Counted { public: CURL* curl_; curl_slist* chunks_; CURLHandle() : curl_{nullptr}, chunks_{nullptr} { pthread_once(&once, init); curl_ = curl_easy_init(); ASSERT(curl_); } ~CURLHandle() { curl_slist_free_all(chunks_); curl_easy_cleanup(curl_); } }; //---------------------------------------------------------------------------------------------------------------------- class EasyCURLResponseImp : public eckit::Counted { public: EasyCURLResponseImp(const std::string& url, CURLHandle* curl); ~EasyCURLResponseImp(); virtual void perform() = 0; virtual bool redirect(std::string& location); virtual size_t writeCallback(const void* ptr, size_t size) = 0; virtual size_t headersCallback(const void* ptr, size_t size); virtual const EasyCURLHeaders& headers() { ensureHeaders(); return headers_; } virtual std::string body() const = 0; virtual unsigned long long contentLength(bool& present) = 0; virtual size_t read(void* ptr, size_t size) = 0; virtual void ensureHeaders() = 0; std::string url_; CURLHandle* ch_; long code_; bool body_; EasyCURLHeaders headers_; virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream& s, const EasyCURLResponseImp& c) { c.print(s); return s; } static size_t _writeCallback(void* ptr, size_t size, size_t nmemb, void* userdata); static size_t _headersCallback(void* ptr, size_t size, size_t nmemb, void* userdata); }; bool EasyCURLResponseImp::redirect(std::string& location) { ensureHeaders(); ASSERT(code_); for (auto c : http_codes) { if (c.code_ == code_) { if (c.redirect_) { char* url = nullptr; _(curl_easy_getinfo(ch_->curl_, CURLINFO_REDIRECT_URL, &url)); ASSERT(url); location = url; return true; } } } return false; } class EasyCURLResponseDirect : public EasyCURLResponseImp { public: std::unique_ptr handle_; EasyCURLResponseDirect(const std::string& url, CURLHandle* curl) : EasyCURLResponseImp(url, curl) {}; void perform() override { _(curl_easy_setopt(ch_->curl_, CURLOPT_URL, url_.c_str())); _(curl_easy_setopt(ch_->curl_, CURLOPT_HEADERFUNCTION, &_headersCallback)); _(curl_easy_setopt(ch_->curl_, CURLOPT_HEADERDATA, this)); _(curl_easy_setopt(ch_->curl_, CURLOPT_WRITEFUNCTION, &_writeCallback)); _(curl_easy_setopt(ch_->curl_, CURLOPT_WRITEDATA, this)); _(curl_easy_perform(ch_->curl_)); } std::string body() const override { if (!handle_) { return ""; } size_t size = handle_->size(); const char* p = reinterpret_cast(handle_->data()); return std::string(p, p + size); } size_t writeCallback(const void* ptr, size_t size) override { if (!handle_) { handle_.reset(new MemoryHandle(1024 * 64, true)); handle_->openForWrite(0); } return handle_->write(ptr, size); } unsigned long long contentLength(bool& present) override { NOTIMP; } size_t read(void* ptr, size_t size) override { NOTIMP; } void ensureHeaders() override {} void print(std::ostream&) const override; }; void EasyCURLResponseDirect::print(std::ostream& s) const { s << "EasyCURLResponseStream[" << body() << ", code=" << code_ << "]"; } class EasyCURLResponseStream : public EasyCURLResponseImp { public: CircularBuffer buffer_; EasyCURLResponseStream(const std::string& url, CURLHandle* curl) : EasyCURLResponseImp(url, curl), buffer_(1024 * 1024) { _(curl_multi_add_handle(multi, ch_->curl_)); } ~EasyCURLResponseStream() { _(curl_multi_remove_handle(multi, ch_->curl_)); } void perform() override { _(curl_easy_setopt(ch_->curl_, CURLOPT_URL, url_.c_str())); _(curl_easy_setopt(ch_->curl_, CURLOPT_HEADERFUNCTION, &_headersCallback)); _(curl_easy_setopt(ch_->curl_, CURLOPT_HEADERDATA, this)); _(curl_easy_setopt(ch_->curl_, CURLOPT_WRITEFUNCTION, &_writeCallback)); _(curl_easy_setopt(ch_->curl_, CURLOPT_WRITEDATA, this)); } size_t writeCallback(const void* ptr, size_t size) override { return buffer_.write(ptr, size); } std::string body() const override { NOTIMP; } unsigned long long contentLength(bool& present) override { ensureHeaders(); present = false; auto j = headers_.find("content-length"); if (j != headers_.end()) { present = true; return Translator()((*j).second); } return 0; } int waitForData() { fd_set fdr, fdw, fdx; struct timeval timeout; int maxfd = -1; long time = -1; FD_ZERO(&fdr); FD_ZERO(&fdw); FD_ZERO(&fdx); _(curl_multi_timeout(multi, &time)); if (time >= 0) { timeout.tv_sec = time / 1000; if (timeout.tv_sec > 1) { timeout.tv_sec = 1; } else { timeout.tv_usec = (time % 1000) * 1000; } } else { timeout.tv_sec = 1; timeout.tv_usec = 0; } _(curl_multi_fdset(multi, &fdr, &fdw, &fdx, &maxfd)); if (maxfd == -1) { timeout = {0, 100 * 1000}; select(0, nullptr, nullptr, nullptr, &timeout); } else { SYSCALL(::select(maxfd + 1, &fdr, &fdw, &fdx, &timeout)); } int active = 0; _(curl_multi_perform(multi, &active)); if (active == 0) { _(curl_easy_getinfo(ch_->curl_, CURLINFO_RESPONSE_CODE, &code_)); } return active; } size_t read(void* ptr, size_t size) override { while (buffer_.length() < size) { if (waitForData() == 0) { break; } } // std::cout << "CAPACITY " << buffer_.size() << std::endl; return buffer_.read(ptr, size); } void ensureHeaders() override { while (!body_) { if (waitForData() == 0) { break; } } } void print(std::ostream&) const override; }; void EasyCURLResponseStream::print(std::ostream& s) const { s << "EasyCURLResponseStream[" << "code=" << code_ << "]"; } EasyCURLResponseImp::EasyCURLResponseImp(const std::string& url, CURLHandle* curl) : url_(url), ch_(curl), code_(0), body_(false) { ch_->attach(); } EasyCURLResponseImp::~EasyCURLResponseImp() { ch_->detach(); } size_t EasyCURLResponseImp::headersCallback(const void* ptr, size_t size) { char* p = (char*)ptr; ASSERT(!body_); ASSERT(size >= 2); ASSERT(p[size - 1] == '\n'); ASSERT(p[size - 2] == '\r'); std::string line(p, p + size - 2); if (line.empty()) { body_ = true; _(curl_easy_getinfo(ch_->curl_, CURLINFO_RESPONSE_CODE, &code_)); } else { std::vector v; Tokenizer parse(":"); parse(line, v); if (v.size() == 2) { headers_[StringTools::lower(v[0])] = StringTools::trim(v[1]); } } return size; } //---------------------------------------------------------------------------------------------------------------------- class EasyCURLHandle : public DataHandle { public: EasyCURLHandle(EasyCURLResponseImp* imp, const std::string& message); ~EasyCURLHandle() override; private: EasyCURLResponseImp* imp_; Timer timer_; double read_; Length total_; Offset position_; std::string message_; void print(std::ostream& s) const override; Length openForRead() override; long read(void*, long) override; void close() override; Length size() override; Length estimate() override; Offset position() override { return position_; } bool canSeek() const override { return false; } }; EasyCURLHandle::EasyCURLHandle(EasyCURLResponseImp* imp, const std::string& message) : imp_(imp), read_(0), message_(message) { imp_->attach(); } EasyCURLHandle::~EasyCURLHandle() { imp_->detach(); } void EasyCURLHandle::print(std::ostream& s) const { s << "EasyCURLHandle[" << imp_->url_ << "]"; } Length EasyCURLHandle::openForRead() { return size(); } Length EasyCURLHandle::size() { bool present = false; Length len = imp_->contentLength(present); if (!present) { throw eckit::BadValue("EasyCURLResponseStream: cannot establish contentLength"); } return len; } Length EasyCURLHandle::estimate() { bool present = false; return imp_->contentLength(present); } long EasyCURLHandle::read(void* ptr, long size) { double now = timer_.elapsed(); size = imp_->read(ptr, size); read_ += timer_.elapsed() - now; total_ += size; position_ += size; return size; } void EasyCURLHandle::close() { // std::cout << "EasyCURLHandle::close " << *imp_ << std::endl; // ASSERT(imp_->code_ == 200); if (!message_.empty()) { Log::info() << message_ << " " << Bytes(total_, read_) << std::endl; } } //---------------------------------------------------------------------------------------------------------------------- EasyCURLResponse::EasyCURLResponse(EasyCURLResponseImp* imp) : imp_(imp) { imp_->attach(); } EasyCURLResponse::~EasyCURLResponse() { imp_->detach(); } EasyCURLResponse::EasyCURLResponse(const EasyCURLResponse& other) : imp_(other.imp_) { imp_->attach(); } EasyCURLResponse& EasyCURLResponse::operator=(const eckit::EasyCURLResponse& other) { if (this == &other) { return *this; } if (imp_ != other.imp_) { imp_->detach(); imp_ = other.imp_; imp_->attach(); } return *this; } Value EasyCURLResponse::json() const { return JSONParser::decodeString(imp_->body()); } std::string EasyCURLResponse::body() const { return imp_->body(); } const EasyCURLHeaders& EasyCURLResponse::headers() const { imp_->ensureHeaders(); return imp_->headers_; } unsigned long long EasyCURLResponse::contentLength() const { bool present = false; return imp_->contentLength(present); } size_t EasyCURLResponse::read(void* ptr, size_t size) const { return imp_->read(ptr, size); } int EasyCURLResponse::code() const { return imp_->code_; } DataHandle* EasyCURLResponse::dataHandle(const std::string& message) const { return new EasyCURLHandle(imp_, message); // return new BufferedHandle(new EasyCURLHandle(imp_, message)); } void EasyCURLResponse::print(std::ostream& out) const { imp_->print(out); } //---------------------------------------------------------------------------------------------------------------------- EasyCURL::EasyCURL() : ch_(new CURLHandle()) { ch_->attach(); } EasyCURL::~EasyCURL() { ch_->detach(); } //---------------------------------------------------------------------------------------------------------------------- void EasyCURL::verbose(bool on) { _(curl_easy_setopt(ch_->curl_, CURLOPT_VERBOSE, on ? 1L : 0L)); } void EasyCURL::followLocation(bool on) { _(curl_easy_setopt(ch_->curl_, CURLOPT_FOLLOWLOCATION, on ? 1L : 0L)); } void EasyCURL::sslVerifyPeer(bool on) { _(curl_easy_setopt(ch_->curl_, CURLOPT_SSL_VERIFYPEER, on ? 1L : 0L)); } void EasyCURL::sslVerifyHost(bool on) { _(curl_easy_setopt(ch_->curl_, CURLOPT_SSL_VERIFYHOST, on ? 1L : 0L)); } void EasyCURL::useSSL(bool use) { _(curl_easy_setopt(ch_->curl_, CURLOPT_USE_SSL, use ? CURLUSESSL_ALL : CURLUSESSL_NONE)); } void EasyCURL::failOnError(bool on) { _(curl_easy_setopt(ch_->curl_, CURLOPT_FAILONERROR, on ? 1L : 0L)); } //---------------------------------------------------------------------------------------------------------------------- EasyCURLResponse EasyCURL::request(const std::string& url, bool stream) { std::string location(url); const size_t max_redirects = 10; for (size_t i = 0; i < max_redirects; ++i) { std::unique_ptr r(nullptr); if (stream) { r.reset(new EasyCURLResponseStream(location, ch_)); } else { r.reset(new EasyCURLResponseDirect(url, ch_)); } r->perform(); if (!r->redirect(location)) { return EasyCURLResponse(r.release()); } } throw SeriousBug("EasyCURL too many redirects for:" + url); } EasyCURLResponse EasyCURL::GET(const std::string& url, bool stream) { _(curl_easy_setopt(ch_->curl_, CURLOPT_CUSTOMREQUEST, NULL)); _(curl_easy_setopt(ch_->curl_, CURLOPT_HTTPGET, 1L)); return request(url, stream); } EasyCURLResponse EasyCURL::HEAD(const std::string& url) { _(curl_easy_setopt(ch_->curl_, CURLOPT_CUSTOMREQUEST, NULL)); _(curl_easy_setopt(ch_->curl_, CURLOPT_NOBODY, 1L)); return request(url); } EasyCURLResponse EasyCURL::POST(const std::string& url, const std::string& data) { _(curl_easy_setopt(ch_->curl_, CURLOPT_CUSTOMREQUEST, NULL)); _(curl_easy_setopt(ch_->curl_, CURLOPT_POST, 1L)); _(curl_easy_setopt(ch_->curl_, CURLOPT_POSTFIELDS, data.c_str())); return request(url); } namespace { /** * Copies the content from input `userdata` into output `buffer` * * Returns the number of characters copied */ size_t readCallback(char* buffer, size_t size [[maybe_unused]], size_t nitems [[maybe_unused]], void* userdata) { auto data = static_cast(userdata); data->copy(buffer, data->size()); return data->size(); } } // namespace EasyCURLResponse EasyCURL::PUT(const std::string& url, const std::string& data) { _(curl_easy_setopt(ch_->curl_, CURLOPT_UPLOAD, 1L)); // Setup callback to read PUT request body (i.e. copy the given 'data' into an internal 'buffer') _(curl_easy_setopt(ch_->curl_, CURLOPT_READFUNCTION, readCallback)); _(curl_easy_setopt(ch_->curl_, CURLOPT_READDATA, &data)); _(curl_easy_setopt(ch_->curl_, CURLOPT_INFILESIZE, data.size())); return request(url); } EasyCURLResponse EasyCURL::DELETE(const std::string& url) { _(curl_easy_setopt(ch_->curl_, CURLOPT_CUSTOMREQUEST, "DELETE")); return request(url); } void EasyCURL::userAgent(const std::string& value) { _(curl_easy_setopt(ch_->curl_, CURLOPT_USERAGENT, value.c_str())); } size_t EasyCURLResponseImp::_writeCallback(void* ptr, size_t size, size_t nmemb, void* userdata) { return reinterpret_cast(userdata)->writeCallback(ptr, size * nmemb); } size_t EasyCURLResponseImp::_headersCallback(void* ptr, size_t size, size_t nmemb, void* userdata) { return reinterpret_cast(userdata)->headersCallback(ptr, size * nmemb); } void EasyCURL::headers(const EasyCURLHeaders& headers) { curl_slist_free_all(ch_->chunks_); for (auto j = headers.begin(); j != headers.end(); ++j) { std::ostringstream oss; oss << (*j).first << ": " << (*j).second; ch_->chunks_ = curl_slist_append(ch_->chunks_, oss.str().c_str()); } if (ch_->chunks_) { _(curl_easy_setopt(ch_->curl_, CURLOPT_HTTPHEADER, ch_->chunks_)); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/TransferWatcher.cc0000664000175000017500000000201115161702250020702 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/TransferWatcher.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- struct DummyTransferWatcher : public TransferWatcher { void watch(const void*, long) {} }; TransferWatcher& TransferWatcher::dummy() { static DummyTransferWatcher x; return x; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/Base64.h0000664000175000017500000000165615161702250016504 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_Base64_h #define eckit_Base64_h #include namespace eckit { class DataHandle; //----------------------------------------------------------------------------- class Base64 { public: // methods Base64(bool url = false); size_t decode(DataHandle& in, DataHandle& out); size_t encode(DataHandle& in, DataHandle& out); private: unsigned char encode_[256]; unsigned char decode_[256]; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/DataHandle.cc0000664000175000017500000003047015161702250017577 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/eckit.h" #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/io/DblBuffer.h" #include "eckit/io/MoverTransfer.h" #include "eckit/log/Bytes.h" #include "eckit/log/Progress.h" #include "eckit/log/Timer.h" #include "eckit/runtime/Metrics.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- AutoClose::~AutoClose() noexcept(false) { bool fail = !Exception::throwing(); try { handle_.close(); } catch (std::exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; if (fail) { Log::error() << "** Exception is re-thrown" << std::endl; throw; } Log::error() << "** An exception is already in progress" << std::endl; Log::error() << "** Exception is ignored" << std::endl; } } //---------------------------------------------------------------------------------------------------------------------- ClassSpec DataHandle::classSpec_ = { &Streamable::classSpec(), "DataHandle", }; Reanimator DataHandle::reanimator_; DataHandle::DataHandle() {} DataHandle::DataHandle(Stream& s) : Streamable(s) {} void DataHandle::encode(Stream& s) const { Streamable::encode(s); } void DataHandle::flush() { std::ostringstream os; os << "DataHandle::flush() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } Length DataHandle::saveInto(DataHandle& other, TransferWatcher& watcher) { static const bool moverTransfer = Resource("-mover;moverTransfer", 0); compress(); Log::status() << Bytes(estimate()) << " " << title() << " => " << other.title() << std::endl; if (moverTransfer && moveable() && other.moveable()) { Log::debug() << "Using MoverTransfer" << std::endl; MoverTransfer mover(watcher); return mover.transfer(*this, other); } static const bool doubleBuffer = Resource("doubleBuffer", 0); if (doubleBuffer && doubleBufferOK() && other.doubleBufferOK()) { static const long bufsize = Resource("doubleBufferSize", 10 * 1024 * 1024 / 20); static const long count = Resource("doubleBufferCount", 20); Metrics::set("double_buffering", true); DblBuffer buf(count, bufsize, watcher); return buf.copy(*this, other); } static const long bufsize = Resource("bufferSize;$ECKIT_DATAHANDLE_SAVEINTO_BUFFER_SIZE", 64 * 1024 * 1024); Buffer buffer(bufsize); watcher.watch(nullptr, 0); Length estimate = openForRead(); AutoClose closer1(*this); watcher.fromHandleOpened(); other.openForWrite(estimate); AutoClose closer2(other); watcher.toHandleOpened(); Progress progress("Moving data", 0, estimate); Length total = 0; long length = -1; double readTime = 0; double lastRead = 0; double writeTime = 0; double lastWrite = 0; Timer timer("Save into"); bool more = true; while (more) { more = false; try { while ((length = read(buffer, buffer.size())) > 0) { double r = timer.elapsed() - lastRead; readTime += r; lastWrite = timer.elapsed(); if (other.write((const char*)buffer, length) != length) { throw WriteError(name() + " into " + other.name()); } double w = timer.elapsed() - lastWrite; writeTime += w; total += length; progress(total); watcher.watch(buffer, length); lastRead = timer.elapsed(); Bytes rRate(total, readTime); Bytes wRate(total, writeTime); Log::message() << rRate.shorten() << " " << wRate.shorten() << std::endl; } } catch (RestartTransfer& retry) { Log::warning() << "Retrying transfer from " << retry.from() << " (" << Bytes(retry.from()) << ")" << std::endl; restartReadFrom(retry.from()); other.restartWriteFrom(retry.from()); watcher.restartFrom(retry.from()); Log::warning() << "Total so far " << total << std::endl; total = Length(0) + retry.from(); Log::warning() << "New total " << total << std::endl; more = true; } } Log::message() << "" << std::endl; Log::info() << "Read rate: " << Bytes(total, readTime) << std::endl; Log::info() << "Write rate: " << Bytes(total, writeTime) << std::endl; if (length < 0) { throw ReadError(name() + " into " + other.name()); } if (estimate != Length(0) && estimate != total) { std::ostringstream os; os << "DataHandle::saveInto got " << total << " bytes out of " << estimate; throw ReadError(name() + " into " + other.name() + " " + os.str()); } this->collectMetrics("source"); other.collectMetrics("target"); Metrics::set("size", total); Metrics::set("time", timer.elapsed()); Metrics::set("read_time", readTime); Metrics::set("write_time", writeTime); Metrics::set("double_buffering", false); return total; } Length DataHandle::saveInto(const PathName& path, TransferWatcher& w) { std::unique_ptr file{path.fileHandle()}; return saveInto(*file, w); } Length DataHandle::copyTo(DataHandle& other, long bufsize, Length maxsize, TransferWatcher& watcher) { if (bufsize == -1) { bufsize = Resource("bufferSize;$ECKIT_DATAHANDLE_COPYTO_BUFFER_SIZE", 64 * 1024 * 1024); } Buffer buffer(bufsize); Length estimate = openForRead(); watcher.fromHandleOpened(); AutoClose closer1(*this); Length toRead = ((maxsize != Length(-1)) ? std::min(estimate, maxsize) : estimate); other.openForWrite(toRead); watcher.toHandleOpened(); AutoClose closer2(other); Length total = 0; long length = -1; while ((toRead <= Length(0) || total < toRead) && (length = read(buffer, toRead <= Length(0) ? bufsize : std::min(bufsize, (long)(toRead - total)))) > 0) { if (other.write((const char*)buffer, length) != length) { throw WriteError(name() + " into " + other.name()); } watcher.watch(buffer, length); total += length; } if (length < 0) { throw ReadError(name() + " into " + other.name()); } if (toRead != Length(0) && toRead != total) { std::ostringstream os; os << "DataHandle::copyTo got " << total << " bytes out of " << toRead; throw ReadError(name() + " into " + other.name() + " " + os.str()); } return total; } std::string DataHandle::name() const { std::ostringstream s; s << *this; return s.str(); } std::string DataHandle::title() const { return className(); } std::string DataHandle::metricsTag() const { return title(); } void DataHandle::collectMetrics(const std::string& what) const { Metrics::set(what, this->metricsTag()); } template <> Streamable* Reanimator::ressucitate(Stream& s) const { return nullptr; } bool DataHandle::compare(DataHandle& other) { size_t bufsize = static_cast(Resource("compareBufferSize", 10 * 1024 * 1024)); Buffer buffer1(bufsize); Buffer buffer2(bufsize); DataHandle& self = *this; Length estimate1 = self.openForRead(); AutoClose closer1(self); Length estimate2 = other.openForRead(); AutoClose closer2(other); if (estimate1 != estimate2) { Log::error() << "DataHandle::compare(" << self << "," << other << ") failed: openForRead() returns " << estimate1 << " and " << estimate2 << std::endl; return false; } Log::status() << "Comparing data" << std::endl; Progress progress("Comparing data", 0, estimate1); Length total = 0; for (;;) { long len1 = self.read(buffer1, buffer1.size()); long len2 = other.read(buffer2, buffer2.size()); if (len1 != len2) { Log::error() << "DataHandle::compare(" << self << "," << other << ") failed: read() returns " << len1 << " and " << len2 << std::endl; return false; } if (len1 <= 0 && len2 <= 0) { Log::info() << "DataHandle::compare(" << self << "," << other << ") is successful" << std::endl; return true; } total += len1; progress(total); if (::memcmp(buffer1, buffer2, len1)) { Log::error() << "DataHandle::compare(" << self << "," << other << ") failed: memcmp() returns non-zero value" << std::endl; return false; } } // should never arrive here } Offset DataHandle::position() { std::ostringstream os; os << "DataHandle::position() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } void DataHandle::rewind() { std::ostringstream os; os << "DataHandle::rewind() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } Offset DataHandle::seek(const Offset& from) { std::ostringstream os; os << "DataHandle::seek(" << from << ") [" << *this << "]"; throw NotImplemented(os.str(), Here()); } bool DataHandle::canSeek() const { std::ostringstream os; os << "DataHandle::canSeek() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } Length DataHandle::estimate() { return 0; } Length DataHandle::openForRead() { std::ostringstream os; os << "DataHandle::openForRead() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } void DataHandle::openForWrite(const Length&) { std::ostringstream os; os << "DataHandle::openForWrite() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } void DataHandle::openForAppend(const Length&) { std::ostringstream os; os << "DataHandle::openForAppend() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } long DataHandle::read(void*, long) { std::ostringstream os; os << "DataHandle::read() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } long DataHandle::write(const void*, long) { std::ostringstream os; os << "DataHandle::write() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } void DataHandle::close() { std::ostringstream os; os << "DataHandle::close() [" << *this << "]"; throw NotImplemented(os.str(), Here()); } Length DataHandle::size() { std::ostringstream oss; oss << "DataHandle::size() [" << *this << "]"; throw NotImplemented(oss.str(), Here()); } void DataHandle::skip(const Length& len) { seek(position() + len); } void DataHandle::restartReadFrom(const Offset& from) { std::ostringstream os; os << "DataHandle::restartReadFrom(" << from << ") [" << *this << "]"; throw NotImplemented(os.str(), Here()); } void DataHandle::restartWriteFrom(const Offset& offset) { std::ostringstream os; os << "DataHandle::restartWriteFrom(" << offset << ") [" << *this << "]"; throw NotImplemented(os.str(), Here()); } void DataHandle::toLocal(Stream& s) const { s << *this; } DataHandle* DataHandle::toLocal() { return this; } void DataHandle::toRemote(Stream& s) const { s << *this; } void DataHandle::selectMover(MoverTransferSelection&, bool) const {} DataHandle* DataHandle::clone() const { std::ostringstream os; os << "DataHandle::clone(" << *this << ")"; throw NotImplemented(os.str(), Here()); } void DataHandle::hash(MD5& md5) const { std::ostringstream os; os << "DataHandle::hash(" << *this << ")"; throw NotImplemented(os.str(), Here()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/AutoCloser.h0000664000175000017500000000210215161702250017523 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @date May 2018 #ifndef eckit_io_AutoCloser_h #define eckit_io_AutoCloser_h #include namespace eckit { template class AutoCloser { T& obj_; public: // methods AutoCloser(T& obj) : obj_(obj) {} /// Assume that close() can throw /// Otherwise we could test the interface with: /// \code{.cpp} /// ~AutoCloser() noexcept(noexcept(std::declval().close())) /// \endcode /// but Intel compiler 17 on Cray XC40 has trouble with it ~AutoCloser() noexcept(false) { obj_.close(); } }; template AutoCloser closer(T& obj) { return AutoCloser(obj); } } // namespace eckit #endif eckit-2.0.7/src/eckit/io/DataHandle.h0000664000175000017500000001231515161702250017437 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// /// @date May 96 #ifndef eckit_io_DataHandle_h #define eckit_io_DataHandle_h #include #include "eckit/filesystem/PathName.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "eckit/io/TransferWatcher.h" #include "eckit/serialisation/Streamable.h" namespace eckit { class MD5; class Metrics; class MoverTransferSelection; //---------------------------------------------------------------------------------------------------------------------- class RestartTransfer { Offset from_; public: RestartTransfer(const Offset& from) : from_(from) {} const Offset& from() const { return from_; } }; //---------------------------------------------------------------------------------------------------------------------- class DataHandle : public Streamable { public: friend std::ostream& operator<<(std::ostream& s, const DataHandle& handle) { handle.print(s); return s; } friend MD5& operator<<(MD5& s, const DataHandle& handle) { handle.hash(s); return s; } // -- Contructors DataHandle(); DataHandle(Stream&); // -- Destructor ~DataHandle() override {} // -- Methods virtual void print(std::ostream& s) const = 0; virtual Length openForRead(); //< Returns estimated length virtual void openForWrite(const Length&); //< Receive estimated length virtual void openForAppend(const Length&); //< Receive estimated length virtual long read(void*, long); virtual long write(const void*, long); virtual void close(); virtual void flush(); virtual Length size(); virtual Length estimate(); virtual Offset position(); virtual Offset seek(const Offset&); virtual bool canSeek() const; virtual void skip(const Length&); virtual void rewind(); virtual void restartReadFrom(const Offset&); virtual void restartWriteFrom(const Offset&); virtual DataHandle* clone() const; /// Save into an other datahandle virtual Length saveInto(DataHandle&, TransferWatcher& = TransferWatcher::dummy()); /// Save into a file virtual Length saveInto(const PathName&, TransferWatcher& = TransferWatcher::dummy()); /// Quiet version of saveInto /// Does not support progess, restart and double buffering virtual Length copyTo(DataHandle&, long bufsize = -1, Length maxsize = -1, TransferWatcher& = TransferWatcher::dummy()); // /// Append to an other datahandle // virtual Length appendTo(DataHandle&, const std::string& metricsPrefix=""); // /// Append to a file // virtual Length appendTo(const PathName&, const std::string& metricsPrefix=""); virtual std::string name() const; /// Create a FILE* from this handle FILE* openf(const char* mode, bool delete_on_close = false); FILE* openf(bool delete_on_close = false); /// Compare bytes bool compare(DataHandle&); // Merge-in an other datahandle virtual bool compress(bool = false) { return false; } virtual bool merge(DataHandle*) { return false; } virtual bool isEmpty() const { return false; } // For remote data movers virtual bool moveable() const { return false; } virtual void toLocal(Stream& s) const; virtual DataHandle* toLocal(); virtual void toRemote(Stream& s) const; virtual void selectMover(MoverTransferSelection&, bool read) const; virtual std::string title() const; virtual std::string metricsTag() const; virtual void collectMetrics(const std::string& what) const; // Tag for metrics collection // This is the MD5 of the Handle, not the data it points to virtual void hash(MD5& md5) const; // This DataHandle can doubleBufferOK virtual bool doubleBufferOK() const { return true; } // -- Overridden methods // From Streamble void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- Class members static ClassSpec classSpec_; static Reanimator reanimator_; }; //---------------------------------------------------------------------------------------------------------------------- class AutoClose { DataHandle& handle_; public: AutoClose(DataHandle& handle) : handle_(handle) {} ~AutoClose() noexcept(false); }; //---------------------------------------------------------------------------------------------------------------------- template <> Streamable* Reanimator::ressucitate(Stream& s) const #ifdef IBM { return 0; } #else ; #endif //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/PeekHandle.h0000664000175000017500000000362115161702250017452 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/PeekHandle.h // Baudouin Raoult - ECMWF May 2020 #ifndef eckit_filesystem_PeekHandle_h #define eckit_filesystem_PeekHandle_h #include #include "eckit/io/HandleHolder.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class PeekHandle : public DataHandle, public HandleHolder { public: /// Contructor, taking ownership PeekHandle(DataHandle*); /// Contructor, not taking ownership PeekHandle(DataHandle&); /// Destructor ~PeekHandle() override; // -- Operators // -- Methods unsigned char peek(size_t); long peek(void* buffer, size_t size, size_t offset = 0); size_t peeked() const; // -- Overridden methods // From DataHandle Length openForRead() override; long read(void*, long) override; void close() override; void rewind() override; void print(std::ostream&) const override; void skip(const Length&) override; Offset seek(const Offset&) override; bool canSeek() const override; Length estimate() override; Offset position() override; Length size() override; std::string title() const override; void collectMetrics(const std::string& what) const override; // Tag for metrics collection private: // members std::deque peek_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/PeekHandle.cc0000664000175000017500000001001715161702250017605 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/PeekHandle.h" #include "eckit/log/Bytes.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- PeekHandle::PeekHandle(DataHandle* h) : HandleHolder(h) {} PeekHandle::PeekHandle(DataHandle& h) : HandleHolder(h) {} PeekHandle::~PeekHandle() {} Length PeekHandle::openForRead() { peek_.clear(); return handle().openForRead(); } void PeekHandle::skip(const Length& len) { if (len == Length(peek_.size())) { peek_.clear(); } else if (len < Length(peek_.size())) { for (long i = 0; i < len; ++i) { peek_.pop_front(); } } else { // This would involve reads/seeks, etc. Not needed now. NOTIMP; } } unsigned char PeekHandle::peek(size_t n) { while (n >= peek_.size()) { unsigned char c; if (handle().read(&c, 1) != 1) { std::ostringstream s; s << handle() << ": failed to read 1 byte"; throw ReadError(s.str(), Here()); } peek_.push_back(c); } return peek_[n]; } long PeekHandle::peek(void* buffer, size_t size, size_t pos) { size_t last = pos + size; unsigned char* buf = static_cast(buffer); while (last > peek_.size()) { long n = std::min(last - peek_.size(), size); long p = handle().read(buf, n); if (p < 0) { std::ostringstream s; s << handle() << ": failed to read " << Bytes(n); throw ReadError(s.str(), Here()); } if (p == 0) { break; } std::copy(buf, buf + p, std::back_inserter(peek_)); } int len = std::min(last, peek_.size()); ASSERT(len >= pos); len -= pos; std::copy(peek_.begin() + pos, peek_.begin() + pos + len, buf); return len; } long PeekHandle::read(void* buffer, long length) { if (peek_.empty()) { return handle().read(buffer, length); } long len = 0; char* p = static_cast(buffer); size_t s = std::min(peek_.size(), size_t(length)); std::copy(peek_.begin(), peek_.begin() + s, p); p += s; len += s; length -= s; while (s--) { peek_.pop_front(); } if (length) { int n = handle().read(p, length); if (n < 0) { std::ostringstream s; s << handle() << ": failed to read " << Bytes(length); throw ReadError(s.str(), Here()); } len += n; } return len; } void PeekHandle::close() { peek_.clear(); handle().close(); } void PeekHandle::rewind() { peek_.clear(); handle().rewind(); } Offset PeekHandle::seek(const Offset& off) { peek_.clear(); return handle().seek(off); } void PeekHandle::print(std::ostream& s) const { s << "PeekHandle["; handle().print(s); s << ']'; } Length PeekHandle::estimate() { return handle().estimate(); } Length PeekHandle::size() { return handle().size(); } bool PeekHandle::canSeek() const { return handle().canSeek(); } Offset PeekHandle::position() { return static_cast(handle().position()) - peek_.size(); } std::string PeekHandle::title() const { return std::string("{") + handle().title() + "}"; } void PeekHandle::collectMetrics(const std::string& what) const { handle().collectMetrics(what); } size_t PeekHandle::peeked() const { return peek_.size(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/SeekableHandle.h0000664000175000017500000000353715161702250020307 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/SeekableHandle.h // Baudouin Raoult - ECMWF May 2020 #ifndef eckit_filesystem_SeekableHandle_h #define eckit_filesystem_SeekableHandle_h #include #include "eckit/io/HandleHolder.h" #include "eckit/io/PeekHandle.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class SeekableHandle : public DataHandle { public: SeekableHandle(PeekHandle*); SeekableHandle(PeekHandle&); /// Destructor ~SeekableHandle() override; // -- Operators // -- Methods // -- Overridden methods // From DataHandle Length openForRead() override; long read(void*, long) override; void close() override; void rewind() override; void print(std::ostream&) const override; void skip(const Length&) override; Offset seek(const Offset&) override; bool canSeek() const override; Length estimate() override; Offset position() override; // Jump to the end of the buffered position, and start again. void clear(); private: // members bool owned_; PeekHandle* handle_; Offset seekableStart_; Offset position_; std::string title() const override; void collectMetrics(const std::string& what) const override; // Tag for metrics collection }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/FileLocker.cc0000664000175000017500000000260615161702250017631 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/FileLocker.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- FileLocker::FileLocker(int fd) : fd_(fd) {} FileLocker::~FileLocker() {} void FileLocker::lockExclusive(off_t off, off_t len) { lockRange(off, len, F_SETLKW, F_WRLCK); } void FileLocker::lockShared(off_t off, off_t len) { lockRange(off, len, F_SETLKW, F_RDLCK); } void FileLocker::unlock(off_t off, off_t len) { lockRange(off, len, F_SETLK, F_UNLCK); } void FileLocker::lockRange(off_t start, off_t len, int cmd, int type) { struct flock lock; lock.l_type = type; lock.l_whence = SEEK_SET; lock.l_start = start; lock.l_len = len; SYSCALL(::fcntl(fd_, cmd, &lock)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/StdioBuf.cc0000664000175000017500000000351315161702250017327 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/io/StdioBuf.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- StdioBuf::StdioBuf(FILE* file) : file_(file) { #ifndef OLD_STREAMBUF /* setg(in_, in_, in_ + sizeof(in_) ); */ setg(in_, in_, in_); setp(out_, out_ + sizeof(out_)); #else setb(in_, in_ + sizeof(in_), 0); setg(in_, in_, in_); setp(out_, out_ + sizeof(out_)); #endif } StdioBuf::~StdioBuf() { sync(); } int StdioBuf::sync() { size_t s = pptr() - pbase(); if (s) { if (::fwrite(pbase(), 1, s, file_) != s) { return EOF; } } setp(pbase(), epptr()); return 0; } int StdioBuf::overflow(int c) { if (sync()) { return EOF; } if (c == EOF) { return 0; } sputc(c); return 0; } int StdioBuf::underflow() { if (gptr() < egptr()) { return *(unsigned char*)gptr(); } #ifndef OLD_STREAMBUF int n = ::fread(in_, 1, sizeof(in_), file_); #else int n = ::fread(base(), 1, sizeof(in_), file_); #endif if (n == EOF || n == 0) { return EOF; } #ifndef OLD_STREAMBUF setg(in_, in_, in_ + n); #else setg(eback(), base(), base() + n); #endif return *(unsigned char*)gptr(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/MultiSocketHandle.h0000664000175000017500000000400115161702250021022 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/MultiSocketHandle.h // Baudouin Raoult - ECMWF Apr 21 #ifndef eckit_filesystem_MultiSocketHandle_h #define eckit_filesystem_MultiSocketHandle_h #include #include "eckit/io/DataHandle.h" #include "eckit/net/MultiSocket.h" namespace eckit { class MultiSocketHandle : public DataHandle { public: // -- Contructors MultiSocketHandle(Stream&); MultiSocketHandle(const std::string& host, int port, size_t streams, size_t messageSize, size_t bufferSize = 0); // -- Destructor ~MultiSocketHandle(); // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void rewind() override; DataHandle* clone() const override; void print(std::ostream&) const override; std::string title() const override; bool moveable() const override { return true; } bool canSeek() const override { return false; } // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } protected: std::string host_; int port_ = 0; std::unique_ptr connection_; size_t streams_ = 0; size_t messageSize_ = 0; size_t bufferSize_ = 0; private: static ClassSpec classSpec_; static Reanimator reanimator_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/StatsHandle.h0000664000175000017500000000553215161702250017667 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Partio/FileHandle.h // Baudouin Raoult - ECMWF Decembre 2013 #ifndef eckit_filesystem_StatsHandle_h #define eckit_filesystem_StatsHandle_h #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/HandleHolder.h" #include "eckit/log/Timer.h" #include "eckit/types/Types.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class StatsHandle : public DataHandle, public HandleHolder { public: // -- Contructors StatsHandle(DataHandle& handle); StatsHandle(DataHandle* handle); // -- Destructor ~StatsHandle(); // -- Methods // -- Overridden methods // From DataHandle void print(std::ostream& s) const override; Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; Length estimate() override; Offset position() override; Offset seek(const Offset&) override; bool canSeek() const override { return handle().canSeek(); } void skip(const Length&) override; void rewind() override; void restartReadFrom(const Offset&) override; void restartWriteFrom(const Offset&) override; DataHandle* clone() const override; using DataHandle::saveInto; Length saveInto(DataHandle& other, TransferWatcher& watcher) override; std::string name() const override; bool compress(bool) override; bool merge(DataHandle*) override; bool isEmpty() const override; bool moveable() const override; void toLocal(Stream& s) const override; DataHandle* toLocal() override; void toRemote(Stream& s) const override; void selectMover(MoverTransferSelection&, bool read) const override; std::string title() const override; void collectMetrics(const std::string& what) const override; // Tag for metrics collection private: // -- Members size_t reads_; size_t seeks_; // size_t positions_; // unused size_t writes_; unsigned long long bytesRead_; unsigned long long bytesWritten_; Timer timer_; double readTime_; double writeTime_; double seekTime_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/FileHandle.cc0000664000175000017500000001670215161702250017607 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/eckit.h" #include "eckit/config/Resource.h" #include "eckit/io/DataHandle.h" #include "eckit/io/FDataSync.h" #include "eckit/io/FileHandle.h" #include "eckit/io/MoverTransferSelection.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/os/Stat.h" #include "eckit/utils/MD5.h" namespace eckit { ClassSpec FileHandle::classSpec_ = { &DataHandle::classSpec(), "FileHandle", }; Reanimator FileHandle::reanimator_; void FileHandle::print(std::ostream& s) const { s << "FileHandle[file=" << name_ << ']'; } void FileHandle::encode(Stream& s) const { DataHandle::encode(s); s << name_; s << overwrite_; } FileHandle::FileHandle(Stream& s) : DataHandle(s), overwrite_(false), file_{nullptr}, read_(false) { s >> name_; s >> overwrite_; } FileHandle::FileHandle(const std::string& name, bool overwrite) : name_(name), overwrite_(overwrite), file_{nullptr}, read_(false) {} FileHandle::~FileHandle() {} void FileHandle::open(const char* mode) { file_ = ::fopen(name_.c_str(), mode); if (file_ == nullptr) { throw CantOpenFile(name_); } // Don't buffer writes, so we know when the filesystems // are full at fwrite time, and not at fclose time. // There should not be any performances issues as // this class is used with large buffers anyway if (!(::strcmp(mode, "r") == 0)) { setbuf(file_, nullptr); } else { static long bufSize = Resource("FileHandleIOBufferSize;$FILEHANDLE_IO_BUFFERSIZE;-FileHandleIOBufferSize", 0); long size = bufSize; if (size) { Log::debug() << "FileHandle using " << Bytes(size) << std::endl; buffer_.reset(new Buffer(size)); Buffer& b = *(buffer_.get()); ::setvbuf(file_, b, _IOFBF, size); } } // Log::info() << "FileHandle::open " << name_ << " " << mode << " " << fileno(file_) << std::endl; } Length FileHandle::openForRead() { read_ = true; open("r"); return estimate(); } void FileHandle::openForWrite(const Length& length) { read_ = false; PathName path(name_); // This is for preallocated files if (overwrite_ && path.exists()) { ASSERT(length == path.size()); open("r+"); } else { open("w"); } } void FileHandle::openForAppend(const Length&) { open("a"); } long FileHandle::read(void* buffer, long length) { return ::fread(buffer, 1, length, file_); } long FileHandle::write(const void* buffer, long length) { ASSERT(buffer); errno = 0; long written = ::fwrite(buffer, 1, length, file_); if (written != length && errno == ENOSPC) { long len = written; do { ::clearerr(file_); Log::status() << "Disk is full, waiting 1 minute ..." << std::endl; ::sleep(60); errno = 0; buffer = ((char*)buffer) + len; length -= len; len = ::fwrite(buffer, 1, length, file_); written += len; } while (len != length && errno == ENOSPC); } return written; } void FileHandle::flush() { if (file_ == nullptr) { return; } if (file_) { if (!read_) { if (::fflush(file_)) { throw WriteError(std::string("fflush(") + name_ + ")", Here()); } int ret = eckit::fsync(fileno(file_)); if (ret < 0) { std::ostringstream oss; oss << "Cannot fsync(" << name_ << ") " << fileno(file_); throw FailedSystemCall(oss.str()); } // On Linux, you must also flush the directory static bool fileHandleSyncsParentDir = eckit::Resource("fileHandleSyncsParentDir", true); if (fileHandleSyncsParentDir) { PathName(name_).syncParentDirectory(); } } } } void FileHandle::close() { if (file_ == nullptr) { return; } if (file_) { // The OS may have large system buffers, therefore the close may be successful without the // data being physicaly on disk. If there is a power failure, we lose some data. // So we need to fsync flush(); if (::fclose(file_) != 0) { throw WriteError(std::string("fclose ") + name()); } } else { Log::warning() << "Closing FileHandle " << name_ << ", file is not opened" << std::endl; } buffer_.reset(); file_ = nullptr; } void FileHandle::rewind() { ::rewind(file_); } Length FileHandle::size() { Stat::Struct info; SYSCALL(Stat::stat(name_.c_str(), &info)); return info.st_size; } Length FileHandle::estimate() { Stat::Struct info; SYSCALL(Stat::stat(name_.c_str(), &info)); return info.st_size; } bool FileHandle::isEmpty() const { Stat::Struct info; if (Stat::stat(name_.c_str(), &info) == -1) { return false; // File does not exists } return info.st_size == 0; } Offset FileHandle::position() { ASSERT(file_); return ::ftello(file_); } void FileHandle::advance(const Length& len) { off_t l = len; if (::fseeko(file_, l, SEEK_CUR) < 0) { throw ReadError(name_); } } void FileHandle::restartReadFrom(const Offset& from) { ASSERT(read_); Log::warning() << *this << " restart read from " << from << std::endl; off_t l = from; if (::fseeko(file_, l, SEEK_SET) < 0) { throw ReadError(name_); } ASSERT(::ftello(file_) == l); } void FileHandle::restartWriteFrom(const Offset& from) { ASSERT(!read_); Log::warning() << *this << " restart write from " << from << std::endl; off_t l = from; if (::fseeko(file_, l, SEEK_SET) < 0) { throw ReadError(name_); } ASSERT(::ftello(file_) == l); } Offset FileHandle::seek(const Offset& from) { off_t l = from; if (::fseeko(file_, l, SEEK_SET) < 0) { throw ReadError(name_); } off_t w = ::ftello(file_); ASSERT(w == l); return w; } bool FileHandle::canSeek() const { return true; } void FileHandle::skip(const Length& n) { off_t l = n; if (::fseeko(file_, l, SEEK_CUR) < 0) { throw ReadError(name_); } } void FileHandle::toRemote(Stream& s) const { PathName p(PathName(name_).clusterName()); std::unique_ptr remote(p.fileHandle()); s << *remote; } void FileHandle::selectMover(MoverTransferSelection& c, bool read) const { if (read) { c.updateCost(NodeInfo::thisNode(), const_cast(this)->estimate()); } else { // Just mark the node as being a candidate c.updateCost(NodeInfo::thisNode(), 0); } } std::string FileHandle::title() const { return PathName::shorten(name_); } std::string FileHandle::metricsTag() const { return PathName::metricsTag(name_); } DataHandle* FileHandle::clone() const { return new FileHandle(name_, overwrite_); } void FileHandle::hash(MD5& md5) const { md5 << "FileHandle"; md5 << name_; } } // namespace eckit eckit-2.0.7/src/eckit/io/SharedBuffer.h0000664000175000017500000000522615161702250020015 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date July 2017 #ifndef eckit_io_SharedBuffer_h #define eckit_io_SharedBuffer_h #include #include #include "eckit/io/Buffer.h" #include "eckit/memory/Counted.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// A buffer that can be shared and is thread-safe /// Note that this buffer, once shared should be for read-only shared-access (at the moment) /// because access is not controlled by locking, only allocation and deallocation class CountedBuffer : public eckit::Buffer, public eckit::Counted { public: // methods CountedBuffer(size_t size) : Buffer(size) {} }; //---------------------------------------------------------------------------------------------------------------------- class SharedBuffer { public: // methods SharedBuffer(size_t size); SharedBuffer(CountedBuffer* b); ~SharedBuffer() { buffer_->detach(); } SharedBuffer(const SharedBuffer& s) : buffer_(s.buffer_) { buffer_->attach(); } SharedBuffer& operator=(const SharedBuffer& s) { if (&s != this) { buffer_->detach(); buffer_ = s.buffer_; buffer_->attach(); } return *this; } CountedBuffer* operator->() const { return buffer_; } void* data() { return buffer_->data(); } size_t size() const { return buffer_->size(); } // void resize(size_t size) { buffer_->resize(size); } /// Careful, use str() to convert the contents of a buffer to a string /// and don't rely on the contents to be null terminated /// This copies the data std::string str() const { return std::string(*buffer_, size()); } operator const Buffer&() const { return *buffer_; } operator Buffer&() { return *buffer_; } private: // methods void print(std::ostream& os) const; friend std::ostream& operator<<(std::ostream& s, const SharedBuffer& o) { o.print(s); return s; } private: // members CountedBuffer* buffer_; ///< @invariant always a valid pointer }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/FileBase.h0000664000175000017500000000405415161702250017125 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FileBase.h // Baudouin Raoult - ECMWF Jun 97 #ifndef eckit_FileBase_h #define eckit_FileBase_h #include "eckit/eckit.h" #include "eckit/filesystem/PathName.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- template class FileBase { public: // -- Exceptions // None // -- Contructors FileBase(const PathName&); // -- Destructor ~FileBase(); // -- Convertors // None // -- Operators // None // -- Methods bool read(long, T&); void write(long, const T&); // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods // void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed FileBase(const FileBase&); FileBase& operator=(const FileBase&); // -- Members struct Record { T data_; bool valid_; }; int fd_; PathName path_; Record buffer_; off_t pos_; // -- Methods // None // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FileBase& p) // { p.print(s); return s; } }; //----------------------------------------------------------------------------- } // namespace eckit #include "eckit/io/FileBase.cc" #endif eckit-2.0.7/src/eckit/io/MoverTransferSelection.cc0000664000175000017500000000732115161702250022254 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/MoverTransferSelection.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/cluster/ClusterNodes.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/runtime/Metrics.h" namespace eckit { MoverTransferSelection::MoverTransferSelection() {} MoverTransferSelection::~MoverTransferSelection() {} void MoverTransferSelection::updateCost(const std::string& name, const eckit::Length& length) { cost_[name] += length; } void MoverTransferSelection::updateCost(const eckit::NodeInfo& node, const eckit::Length& length) { updateCost(node.node(), length); } void MoverTransferSelection::requiredMoverAttributes(const std::set& attrs) { moverAttributes_.insert(attrs.begin(), attrs.end()); } void MoverTransferSelection::selectedMover(NodeInfo& result, bool& metrics) { metrics = false; if (preferredMover_.length()) { if (ClusterNodes::available("mover", preferredMover_)) { NodeInfo preferred = ClusterNodes::lookUp("mover", preferredMover_); if (preferred.supportsAttributes(moverAttributes_)) { Log::info() << "Using preferred mover " << preferredMover_ << std::endl; result = preferred; return; } Log::warning() << "Preferred mover " << preferredMover_ << " does not support mover attributes: " << moverAttributes_ << std::endl; } else { Log::warning() << "Preferred mover " << preferredMover_ << " is not available" << std::endl; } } std::map cost; for (const auto& c : cost_) { if (ClusterNodes::available("mover", c.first) && ClusterNodes::lookUp("mover", c.first).supportsAttributes(moverAttributes_)) { cost.insert(c); } } if (cost.empty()) { result = ClusterNodes::any("mover", moverAttributes_); return; } Log::info() << "MoverTransfer::transfer cost:" << std::endl; for (std::map::iterator j = cost.begin(); j != cost.end(); ++j) { Log::info() << " " << (*j).first << " => " << Bytes((*j).second) << std::endl; } std::string which; Length best = 0; for (const auto& c : cost) { if (c.second >= best) { best = c.second; which = c.first; } } ASSERT(which != ""); metrics = true; result = ClusterNodes::lookUp("mover", which); } NodeInfo MoverTransferSelection::selectedMover() { NodeInfo result; bool metrics = false; selectedMover(result, metrics); Metrics::set("mover_node", result.node()); if (metrics) { for (auto j = cost_.begin(); j != cost_.end(); ++j) { std::string h = (*j).first; unsigned long long l = (*j).second; Metrics::set("mover_costs." + h, l); } } return result; } void MoverTransferSelection::preferredMover(const NodeInfo& node) { preferredMover(node.node()); } void MoverTransferSelection::preferredMover(const std::string& name) { Log::info() << "MoverTransferSelection::preferredMover " << name << std::endl; preferredMover_ = name; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/FileDescHandle.h0000664000175000017500000000326715161702250020252 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/FileDescHandle.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_filesystem_FileDescHandle_h #define eckit_filesystem_FileDescHandle_h #include "eckit/io/DataHandle.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class FileDescHandle : public DataHandle { public: // -- Contructors FileDescHandle(int, bool close = false); // -- Destructor ~FileDescHandle(); // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void print(std::ostream&) const override; Offset position() override; Offset seek(const Offset&) override; bool canSeek() const override { return true; } void skip(const Length&) override; // From Streamable void encode(Stream&) const override; // -- Class methods private: // -- Members int fd_; bool close_; // -- Class members }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/AIOHandle.h0000664000175000017500000000306615161702250017201 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_io_AIOHandle_h #define eckit_io_AIOHandle_h #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" namespace eckit { struct AIOBuffer; class AIOHandle : public DataHandle { public: // methods AIOHandle(const PathName& path, size_t count = 16, size_t buffsize = 1024 * 1024, bool fsync = false); ~AIOHandle() override; Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; void print(std::ostream&) const override; Length size() override; Length estimate() override; Offset position() override; bool canSeek() const override { return false; } private: // methods size_t getFreeSlot(); protected: // members PathName path_; private: // members std::vector buffers_; size_t used_; size_t count_; int fd_; off_t pos_; bool fsync_; std::string title() const override; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/AsyncHandle.h0000664000175000017500000000517315161702250017647 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/AsyncHandle.h // Baudouin Raoult - ECMWF Dec 17 #ifndef eckit_io_AsyncHandle_h #define eckit_io_AsyncHandle_h #include #include "eckit/io/Buffer.h" #include "eckit/io/HandleHolder.h" #include "eckit/thread/MutexCond.h" #include "eckit/thread/ThreadControler.h" namespace eckit { //----------------------------------------------------------------------------- class AsyncHandle : public DataHandle, public HandleHolder { public: /// Contructor, taking ownership AsyncHandle(DataHandle*, size_t = 1024 * 1024, size_t rounding = 64 * 1024); /// Contructor, not taking ownership AsyncHandle(DataHandle&, size_t = 1024 * 1024, size_t rounding = 64 * 1024); /// Destructor ~AsyncHandle() override; // -- Operators // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; void print(std::ostream&) const override; void skip(const Length&) override; Offset seek(const Offset&) override; Length estimate() override; Offset position() override; DataHandle* clone() const override; // From Streamable #if 0 void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } #endif // -- Class methods #if 0 static const ClassSpec& classSpec() { return classSpec_;} #endif private: // methods private: // members std::string message_; size_t maxSize_; size_t used_; size_t rounding_; bool error_; MutexCond cond_; std::deque > buffers_; ThreadControler thread_; // must be last std::string title() const override; void collectMetrics(const std::string& what) const override; // -- Class members friend class AsyncHandleWriter; #if 0 static ClassSpec classSpec_; static Reanimator reanimator_; #endif }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/EasyCURL.h0000664000175000017500000000535215161702250017044 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date June 2017 #ifndef eckit_io_EasyCURL_h #define eckit_io_EasyCURL_h #include #include "eckit/io/MemoryHandle.h" namespace eckit { class Value; class EasyCURLResponseImp; class CURLHandle; using EasyCURLHeaders = std::map; //---------------------------------------------------------------------------------------------------------------------- class EasyCURLResponse { mutable EasyCURLResponseImp* imp_; public: EasyCURLResponse(EasyCURLResponseImp*); ~EasyCURLResponse(); EasyCURLResponse(const EasyCURLResponse&); EasyCURLResponse& operator=(const EasyCURLResponse&); Value json() const; std::string body() const; const EasyCURLHeaders& headers() const; int code() const; unsigned long long contentLength() const; size_t read(void* ptr, size_t size) const; eckit::DataHandle* dataHandle(const std::string& message = "") const; private: void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const EasyCURLResponse& c) { c.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- class EasyCURL { public: EasyCURL(); ~EasyCURL(); EasyCURLResponse GET(const std::string& url, bool stream = false); EasyCURLResponse HEAD(const std::string& url); EasyCURLResponse PUT(const std::string& url, const std::string& data); EasyCURLResponse POST(const std::string& url, const std::string& data); EasyCURLResponse DELETE(const std::string& url); void verbose(bool on); void followLocation(bool on); void sslVerifyPeer(bool on); void sslVerifyHost(bool on); void failOnError(bool on); void useSSL(bool use); void customRequest(const std::string&); void headers(const EasyCURLHeaders& headers); void userAgent(const std::string&); public: void print(std::ostream&) const; private: CURLHandle* ch_; EasyCURLResponse request(const std::string& url, bool stream = false); friend std::ostream& operator<<(std::ostream& s, const EasyCURL& c) { c.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/MultiHandle.h0000664000175000017500000000607615161702250017667 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Emanuele Danovaro /// @author Simon Smart /// @author Tiago Quintino /// @date May 96 #ifndef eckit_filesystem_MultiHandle_h #define eckit_filesystem_MultiHandle_h #include "eckit/io/DataHandle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class MultiHandle : public DataHandle { public: using HandleList = std::vector; // -- Contructors MultiHandle(); MultiHandle(const LengthList&); MultiHandle(const HandleList&); MultiHandle(const HandleList&, const LengthList&); MultiHandle(Stream&); // -- Destructor ~MultiHandle() override; // -- Operators virtual void operator+=(DataHandle*); virtual void operator+=(const Length&); // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; void print(std::ostream&) const override; void restartReadFrom(const Offset& from) override; Offset position() override; Offset seek(const Offset&) override; bool canSeek() const override; bool merge(DataHandle*) override; bool compress(bool = false) override; Length size() override; Length estimate() override; void toRemote(Stream&) const override; void toLocal(Stream&) const override; DataHandle* toLocal() override; void selectMover(MoverTransferSelection&, bool read) const override; std::string title() const override; bool moveable() const override; DataHandle* clone() const override; void collectMetrics(const std::string& what) const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- Members HandleList datahandles_; HandleList::iterator current_; LengthList::iterator curlen_; LengthList length_; Length written_; mutable std::set requiredAttributes_; bool read_; // -- Methods void openCurrent(); void open(); long read1(char*, long); // -- Class members static ClassSpec classSpec_; static Reanimator reanimator_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/URLHandle.cc0000664000175000017500000000424515161702250017371 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/io/URLHandle.h" #include "eckit/io/EasyCURL.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec URLHandle::classSpec_ = { &DataHandle::classSpec(), "URLHandle", }; Reanimator URLHandle::reanimator_; void URLHandle::print(std::ostream& s) const { s << "URLHandle[uri=" << uri_ << ']'; } void URLHandle::encode(Stream& s) const { DataHandle::encode(s); s << uri_; } URLHandle::URLHandle(Stream& s) : DataHandle(s) { s >> uri_; } URLHandle::URLHandle(const std::string& uri, bool useSSL) : uri_(uri), useSSL_(useSSL) {} URLHandle::~URLHandle() {} Length URLHandle::estimate() { return handle().estimate(); } Length URLHandle::size() { return handle().size(); } Length URLHandle::openForRead() { return handle().estimate(); } void URLHandle::openForWrite(const Length& length) { handle().openForWrite(length); } void URLHandle::openForAppend(const Length& length) { handle().openForAppend(length); } long URLHandle::read(void* buffer, long length) { return handle().read(buffer, length); } long URLHandle::write(const void* buffer, long length) { return handle().write(buffer, length); } void URLHandle::close() { return handle().close(); } DataHandle& URLHandle::handle() { if (!handle_) { EasyCURL curl; /// IDEA: make the options more generic, eg. EasyCURL curl(opts) // curl.followLocation(true); curl.useSSL(useSSL_); handle_.reset(curl.GET(uri_, true).dataHandle()); } return *handle_.get(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/BufferedHandle.cc0000664000175000017500000001144115161702250020445 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/BufferedHandle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- BufferedHandle::BufferedHandle(DataHandle* h, size_t size, bool opened) : HandleHolder(h), buffer_(size), pos_(0), size_(size), used_(0), eof_(false), read_(false), position_(0), opened_(opened) {} BufferedHandle::BufferedHandle(DataHandle& h, size_t size, bool opened) : HandleHolder(h), buffer_(size), pos_(0), size_(size), used_(0), eof_(false), read_(false), position_(0), opened_(opened) {} BufferedHandle::~BufferedHandle() {} Length BufferedHandle::openForRead() { read_ = true; used_ = pos_ = 0; eof_ = false; position_ = 0; if (opened_) { return handle().estimate(); } return handle().openForRead(); } void BufferedHandle::openForWrite(const Length& length) { read_ = false; pos_ = 0; position_ = 0; handle().openForWrite(length); } void BufferedHandle::openForAppend(const Length&) { NOTIMP; } void BufferedHandle::skip(const Length& len) { ASSERT(read_); unsigned long long left = used_ - pos_; unsigned long long n = len; if (n < left) { position_ += n; pos_ += n; return; } seek(position() + len); } long BufferedHandle::read(void* buffer, long length) { long len = 0; long size = length; char* buf = (char*)buffer; ASSERT(read_); if (eof_) { return 0; } while (len < length && !eof_) { long left = used_ - pos_; ASSERT(left >= 0); if (left == 0 && !eof_) { // read() is supposed to return a non-negative number used_ = handle().read(buffer_, size_); pos_ = 0; if (used_ <= 0) { eof_ = true; if (len > 0) { position_ += len; } return len; } left = used_; } char* p = buffer_; long s = size < left ? size : left; ::memcpy(buf + len, p + pos_, s); len += s; ASSERT(len <= length); pos_ += s; ASSERT(pos_ <= used_); size -= s; ASSERT(size >= 0); } if (len > 0) { position_ += len; } return len; } long BufferedHandle::write(const void* buffer, long length) { long written = 0; ASSERT(!read_); while (length > 0) { long left = size_ - pos_; ASSERT(left > 0); size_t len = std::min(left, length); ASSERT(len > 0); char* p = buffer_; const char* q = static_cast(buffer); ::memcpy(p + pos_, q + written, len); pos_ += len; written += len; length -= len; ASSERT(length >= 0); ASSERT(pos_ <= size_); if (pos_ == size_) { bufferFlush(); } } position_ += written; return written; } void BufferedHandle::close() { if (!read_) { bufferFlush(); } handle().close(); } void BufferedHandle::flush() { bufferFlush(); handle().flush(); } void BufferedHandle::rewind() { position_ = 0; used_ = pos_ = 0; eof_ = false; handle().rewind(); } Offset BufferedHandle::seek(const Offset& off) { used_ = pos_ = 0; eof_ = false; position_ = handle().seek(off); return position_; } void BufferedHandle::print(std::ostream& s) const { s << "BufferedHandle["; handle().print(s); s << ']'; } Length BufferedHandle::estimate() { return handle().estimate(); } Offset BufferedHandle::position() { // ASSERT(read_); return position_; } void BufferedHandle::bufferFlush() { if (pos_) { long len = handle().write(buffer_, pos_); ASSERT((size_t)len == pos_); pos_ = 0; } } std::string BufferedHandle::title() const { return std::string("{") + handle().title() + "}"; } void BufferedHandle::collectMetrics(const std::string& what) const { handle().collectMetrics(what); } DataHandle* BufferedHandle::clone() const { return new BufferedHandle(handle().clone(), buffer_.size()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/AsyncHandle.cc0000664000175000017500000001201215161702250017773 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/AsyncHandle.h" #include "eckit/maths/Functions.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Thread.h" namespace eckit { #if 0 ClassSpec AsyncHandle::classSpec_ = {&DataHandle::classSpec(), "AsyncHandle",}; Reanimator AsyncHandle::reanimator_; #endif class AsyncHandleWriter : public Thread { AsyncHandle& owner_; virtual void run(); public: AsyncHandleWriter(AsyncHandle& owner) : owner_(owner) {} }; void AsyncHandleWriter::run() { while (!stopped()) { Buffer* delme = nullptr; try { AutoLock lock(owner_.cond_); while (owner_.buffers_.empty() && !stopped()) { owner_.cond_.wait(); } if (stopped()) { break; } ASSERT(!owner_.buffers_.empty()); std::pair p = owner_.buffers_.front(); owner_.buffers_.pop_front(); owner_.used_ -= p.second->size(); delme = p.second; long written = owner_.handle().write(p.second->data(), p.first); if (written != static_cast(p.first)) { std::ostringstream oss; oss << "AsyncHandleWriter: written " << written << " out of " << p.first << Log::syserr; throw WriteError(oss.str()); } owner_.cond_.signal(); } catch (std::exception& e) { eckit::Log::error() << "AsyncHandleWriter got an exception: " << e.what() << " " << owner_ << std::endl; AutoLock lock(owner_.cond_); owner_.error_ = e.what(); owner_.cond_.signal(); } delete delme; delme = nullptr; } } AsyncHandle::AsyncHandle(DataHandle* h, size_t maxSize, size_t rounding) : HandleHolder(h), maxSize_(maxSize), used_(0), rounding_(rounding), error_(false), thread_(new AsyncHandleWriter(*this), false) { thread_.start(); } AsyncHandle::AsyncHandle(DataHandle& h, size_t maxSize, size_t rounding) : HandleHolder(h), maxSize_(maxSize), used_(0), rounding_(rounding), error_(false), thread_(new AsyncHandleWriter(*this), false) { thread_.start(); } AsyncHandle::~AsyncHandle() { { AutoLock lock(cond_); while (!buffers_.empty()) { delete buffers_.front().second; buffers_.pop_front(); } cond_.signal(); } thread_.stop(); cond_.signal(); thread_.wait(); } Length AsyncHandle::openForRead() { NOTIMP; } void AsyncHandle::openForWrite(const Length& length) { ASSERT(used_ == 0); ASSERT(buffers_.size() == 0); handle().openForWrite(length); } void AsyncHandle::openForAppend(const Length& length) { ASSERT(used_ == 0); ASSERT(buffers_.size() == 0); handle().openForAppend(length); } void AsyncHandle::skip(const Length&) { NOTIMP; } long AsyncHandle::read(void*, long) { NOTIMP; } long AsyncHandle::write(const void* buffer, long length) { AutoLock lock(cond_); size_t size = eckit::round(length, rounding_); while (used_ + size >= maxSize_ && !error_) { // Special case for size > maxSize_ if (buffers_.empty()) { break; } cond_.wait(); } if (error_) { throw WriteError(message_); } std::pair p(length, new Buffer(size)); ::memcpy(p.second->data(), buffer, length); buffers_.push_back(p); used_ += size; cond_.signal(); return length; } void AsyncHandle::close() { flush(); handle().close(); AutoLock lock(cond_); if (error_) { throw WriteError(message_); } } void AsyncHandle::flush() { AutoLock lock(cond_); while (!buffers_.empty() && !error_) { cond_.wait(); } if (error_) { throw WriteError(message_); } handle().flush(); } void AsyncHandle::rewind() { NOTIMP; } Offset AsyncHandle::seek(const Offset&) { NOTIMP; } void AsyncHandle::print(std::ostream& s) const { s << "AsyncHandle["; handle().print(s); s << ']'; } Length AsyncHandle::estimate() { return handle().estimate(); } Offset AsyncHandle::position() { NOTIMP; } std::string AsyncHandle::title() const { return std::string("{") + handle().title() + "}"; } void AsyncHandle::collectMetrics(const std::string& what) const { handle().collectMetrics(what); } DataHandle* AsyncHandle::clone() const { return new AsyncHandle(handle().clone(), maxSize_, rounding_); } } // namespace eckit eckit-2.0.7/src/eckit/io/TCPSocketHandle.cc0000664000175000017500000000453315161702250020526 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/TCPSocketHandle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- void InstantTCPSocketHandle::print(std::ostream& s) const { s << "InstantTCPSocketHandle[]"; } InstantTCPSocketHandle::InstantTCPSocketHandle(net::TCPSocket& s) : connection_(s), read_(true), position_(0) {} InstantTCPSocketHandle::~InstantTCPSocketHandle() {} Length InstantTCPSocketHandle::openForRead() { read_ = true; position_ = 0; return 0; } void InstantTCPSocketHandle::openForWrite(const Length&) { read_ = false; position_ = 0; } void InstantTCPSocketHandle::openForAppend(const Length&) { NOTIMP; } long InstantTCPSocketHandle::read(void* buffer, long length) { long n = connection_.read(buffer, length); if (n > 0) { position_ += n; } return n; } long InstantTCPSocketHandle::write(const void* buffer, long length) { long n = connection_.write(buffer, length); if (n > 0) { position_ += n; } return n; } void InstantTCPSocketHandle::close() {} void InstantTCPSocketHandle::rewind() { seek(0); } Offset InstantTCPSocketHandle::seek(const Offset& o) { ASSERT(read_); if (o < position_) { NOTIMP; } while (position_ < o) { char c[10240]; read(c, std::min(sizeof(c), size_t(o - position_))); } return o; } //---------------------------------------------------------------------------------------------------------------------- TCPSocketHandle::TCPSocketHandle(net::TCPSocket& socket) : InstantTCPSocketHandle(socket), socket_(socket) // Will take onwership {} void TCPSocketHandle::print(std::ostream& s) const { s << "TCPSocketHandle[]"; } void TCPSocketHandle::close() { connection_.close(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/Compress.h0000664000175000017500000000163515161702250017250 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_Compress_h #define eckit_Compress_h #include namespace eckit { class DataHandle; //----------------------------------------------------------------------------- class Compress { public: // methods Compress(size_t maxBits = 16); // -- size_t decode(DataHandle& in, DataHandle& out); size_t encode(DataHandle& in, DataHandle& out); private: const size_t maxBits_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/PartFileHandle.h0000664000175000017500000000544215161702250020277 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Emanuele Danovaro /// @author Simon Smart /// @author Tiago Quintino /// @date May 1996 #ifndef eckit_filesystem_PartFileHandle_h #define eckit_filesystem_PartFileHandle_h #include #include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" #include "eckit/types/Types.h" namespace eckit { class PooledHandle; //---------------------------------------------------------------------------------------------------------------------- class PartFileHandle : public DataHandle { public: // methods PartFileHandle(const PathName&, const OffsetList&, const LengthList&); PartFileHandle(const PathName&, const Offset&, const Length&); explicit PartFileHandle(Stream&); ~PartFileHandle() override; const PathName& path() const { return path_; } // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void rewind() override; void print(std::ostream&) const override; bool merge(DataHandle*) override; bool compress(bool = false) override; Length size() override; Length estimate() override; void restartReadFrom(const Offset& from) override; Offset position() override; Offset seek(const Offset&) override; bool canSeek() const override; void selectMover(MoverTransferSelection&, bool read) const override; std::string title() const override; std::string metricsTag() const override; bool moveable() const override { return true; } DataHandle* clone() const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // members PathName path_; std::unique_ptr handle_; long long pos_; Ordinal index_; OffsetList offset_; LengthList length_; private: // methods long read1(char*, long); static ClassSpec classSpec_; static Reanimator reanimator_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/DblBuffer.cc0000664000175000017500000002271315161702250017446 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/DblBuffer.h" #include "eckit/io/AutoCloser.h" #include "eckit/io/Buffer.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/log/Progress.h" #include "eckit/log/Timer.h" #include "eckit/runtime/Metrics.h" #include "eckit/runtime/Monitor.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/MutexCond.h" #include "eckit/thread/Thread.h" #include "eckit/thread/ThreadControler.h" #include "eckit/value/Value.h" namespace eckit { class DblBufferError : public Exception { public: DblBufferError(const std::string& what) { reason(std::string("Double buffer error: ") + what); } }; struct OneBuffer { MutexCond cond_; bool full_; long length_; char* buffer_; OneBuffer() : full_(false), length_(0), buffer_{nullptr} {} }; class DblBufferTask : public Thread { DblBuffer& owner_; DataHandle& out_; Length estimate_; OneBuffer* buffers_; long parent_; public: DblBufferTask(DataHandle&, DblBuffer&, OneBuffer*, const Length&, long parent); virtual void run(); }; DblBuffer::DblBuffer(long count, long size, TransferWatcher& watcher) : count_(count), bufSize_(size), error_(false), restart_(false), restartFrom_(0), watcher_(watcher) { Log::info() << "Double buffering: " << count_ << " buffers of " << Bytes(size) << " is " << Bytes(count * size) << std::endl; } DblBuffer::~DblBuffer() {} inline void DblBuffer::error(const std::string& why) { AutoLock lock(mutex_); error_ = true; why_ = why; } inline bool DblBuffer::error() { AutoLock lock(mutex_); return error_; } void DblBuffer::restart(RestartTransfer& retry) { AutoLock lock(mutex_); Log::warning() << "Retrying transfer from " << retry.from() << " (" << Bytes(retry.from()) << ")" << std::endl; error_ = true; restart_ = true; restartFrom_ = retry.from(); } Length DblBuffer::copy(DataHandle& in, DataHandle& out) { Timer timer("Double buffer"); in.compress(); Length estimate = in.openForRead(); AutoClose c1(in); watcher_.fromHandleOpened(); out.openForWrite(estimate); AutoClose c2(out); watcher_.toHandleOpened(); Length total = estimate; Length copied = 0; bool more = true; while (more) { more = false; try { copied = copy(in, out, estimate); Log::info() << "Copied: " << copied << ", estimate: " << estimate << std::endl; if (estimate) { if (copied != estimate) { std::ostringstream os; os << "DblBuffer::copy(), copied: " << copied << ", estimate: " << estimate; throw SeriousBug(os.str()); } ASSERT(copied == estimate); } } catch (RestartTransfer& retry) { Log::warning() << "Retrying transfer from " << retry.from() << " (" << Bytes(retry.from()) << ")" << std::endl; in.restartReadFrom(retry.from()); out.restartWriteFrom(retry.from()); estimate = total - retry.from(); more = true; } } Log::info() << "Transfer rate " << Bytes(estimate, timer) << std::endl; return copied; } Length DblBuffer::copy(DataHandle& in, DataHandle& out, const Length& estimate) { Buffer bigbuf(count_ * bufSize_); OneBuffer* buffers = new OneBuffer[count_]; char* addr = bigbuf; for (int j = 0; j < count_; j++) { buffers[j].buffer_ = addr; addr += bufSize_; } Progress progress("Reading data", 0, estimate); error_ = false; inBytes_ = outBytes_ = 0; ThreadControler thread(new DblBufferTask(out, *this, buffers, estimate, Monitor::instance().self()), false); thread.start(); int i = 0; Timer reader("Double buffer reader"); double rate = 0.; double first = 0.; watcher_.watch(nullptr, 0); while (!error()) { Log::message() << "Wait " << i << std::endl; AutoLock lock(buffers[i].cond_); while (buffers[i].full_) { buffers[i].cond_.wait(); } if (error()) { break; } Log::message() << "Read " << i << std::endl; try { double x = reader.elapsed(); buffers[i].length_ = in.read(buffers[i].buffer_, bufSize_); double s = reader.elapsed() - x; Log::status() << Bytes(estimate) << " at " << Bytes(buffers[i].length_ / s) << "/s" << std::endl; rate += s; if (first == 0.) { first = rate; } watcher_.watch(buffers[i].buffer_, buffers[i].length_); } catch (RestartTransfer& retry) { Log::warning() << "RestartTransfer: Exiting reader thread" << std::endl; buffers[i].length_ = -1; restart(retry); } catch (std::exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; Log::error() << "** Exception is handled" << std::endl; buffers[i].length_ = -1; error(e.what()); } Log::message() << "" << std::endl; buffers[i].full_ = true; if (buffers[i].length_ == 0) { buffers[i].cond_.signal(); break; } if (buffers[i].length_ < 0) { ASSERT(error()); Log::warning() << "Read error... " << why_ << std::endl; buffers[i].cond_.signal(); break; } inBytes_ += buffers[i].length_; progress(inBytes_); buffers[i].cond_.signal(); i++; i %= count_; } Log::info() << "Read done " << Bytes(inBytes_) << std::endl; Log::info() << "Read rate " << Bytes(inBytes_ / rate) << "/s" << std::endl; if (first != rate) { Log::info() << "Read rate no mount " << Bytes(inBytes_ / (rate - first)) << "/s" << std::endl; } thread.wait(); delete[] buffers; if (error_) { if (restart_) { throw RestartTransfer(restartFrom_); } throw DblBufferError(why_); } PANIC(inBytes_ != outBytes_); in.collectMetrics("source"); out.collectMetrics("target"); Metrics::set("size", inBytes_); Metrics::set("read_time", rate); Metrics::set("time", reader.elapsed()); return inBytes_; } DblBufferTask::DblBufferTask(DataHandle& out, DblBuffer& owner, OneBuffer* buffers, const Length& estimate, long parent) : Thread(false), owner_(owner), out_(out), estimate_(estimate), buffers_(buffers), parent_(parent) {} void DblBufferTask::run() { Monitor::instance().parent(parent_); Log::status() << "Double buffering " << Bytes(estimate_) << std::endl; Progress progress("Writing data", 0, estimate_); int i = 0; Timer writer("Double buffer writer"); double rate = 0; double first = 0; while (!owner_.error()) { Log::message() << "Wait " << i << std::endl; AutoLock lock(buffers_[i].cond_); while (!buffers_[i].full_) { buffers_[i].cond_.wait(); } if (owner_.error()) { break; } if (buffers_[i].length_ == 0) { break; } long length = -1; Log::message() << "Write " << i << std::endl; try { double x = writer.elapsed(); length = out_.write(buffers_[i].buffer_, buffers_[i].length_); double s = writer.elapsed() - x; Log::status() << Bytes(buffers_[i].length_ / s) << "/s" << std::endl; rate += s; if (first == 0) { first = rate; } ASSERT(length == buffers_[i].length_); } catch (RestartTransfer& retry) { Log::warning() << "RestartTransfer: Exiting writer thread" << std::endl; length = -1; owner_.restart(retry); } catch (std::exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; Log::error() << "** Exception is handled" << std::endl; length = -1; owner_.error(e.what()); } Log::message() << "" << std::endl; buffers_[i].full_ = false; if (length < 0) { ASSERT(owner_.error()); buffers_[i].cond_.signal(); break; } ASSERT(length == buffers_[i].length_); owner_.outBytes_ += length; progress(owner_.outBytes_); buffers_[i].cond_.signal(); i++; i %= owner_.count_; } Log::info() << "Write done " << Bytes(owner_.outBytes_) << std::endl; Log::info() << "Write rate " << Bytes(owner_.outBytes_ / rate) << "/s" << std::endl; if (rate != first) { Log::info() << "Write rate no mount " << Bytes(owner_.outBytes_ / (rate - first)) << "/s" << std::endl; } Metrics::set("write_time", rate); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/BitIO.h0000664000175000017500000000314615161702250016422 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_BitIO_h #define eckit_BitIO_h #include #include "eckit/eckit.h" //----------------------------------------------------------------------------- namespace eckit { class DataHandle; //----------------------------------------------------------------------------- class BitIO { public: // methods /// Contructor // padded: pad the last bits with zero at end-of-file BitIO(DataHandle& handle, bool padded = false); /// Destructor ~BitIO(); // size_t bitCount() const; // Bits written, read size_t byteCount() const; // Bits written, read void write(size_t code, size_t nbits); //============================= // EOF_MARKER = 0: Throw an exception on end of file // EOF_MARKER = N: return N on end of file size_t read(size_t nbits, size_t EOF_MARKER = 0); size_t readAny(size_t& nbits); private: // members DataHandle& handle_; unsigned long long buffer_; size_t used_; size_t bits_; bool write_; bool eof_; bool padded_; bool opened_; // --- void flush(); // -- Class members }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/FileLock.cc0000664000175000017500000000321715161702250017301 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/FileLock.h" #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/os/AutoUmask.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static int openLock(const PathName& lockFile) { AutoUmask mask(0); int fd = 0; lockFile.dirName().mkdir(0777); SYSCALL2(fd = ::open(lockFile.asString().c_str(), O_CREAT | O_RDWR, 0777), lockFile); return fd; } FileLock::FileLock(const PathName& lockFile, bool unlink_at_destruction) : lockFile_(unlink_at_destruction ? lockFile : ""), fd_(openLock(lockFile)), locker_(fd_) {} FileLock::~FileLock() { ::close(fd_); if (!lockFile_.asString().empty()) { try { lockFile_.unlink(false); } catch (const FailedSystemCall& e) { Log::warning() << "Failed to unlink lock file " << lockFile_ << " -- " << e.what() << std::endl; } } } void FileLock::lock() { locker_.lockExclusive(); } void FileLock::unlock() { locker_.unlock(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/PartHandle.cc0000664000175000017500000001142015161702250017626 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/Resource.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/io/PartFileHandle.h" #include "eckit/io/PartHandle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec PartHandle::classSpec_ = { &DataHandle::classSpec(), "PartHandle", }; Reanimator PartHandle::reanimator_; void PartHandle::print(std::ostream& s) const { /* if(format(s) == Log::compactFormat) s << "PartHandle"; else*/ s << "PartHandle[handle=" << handle() << ",offset=" << offset_ << ",length=" << length_ << ']'; } void PartHandle::encode(Stream& s) const { DataHandle::encode(s); s << handle(); s << offset_; s << length_; } PartHandle::PartHandle(Stream& s) : DataHandle(s), HandleHolder(Reanimator::reanimate(s)), pos_(0), index_(0) { s >> offset_; s >> length_; ASSERT(offset_.size() == length_.size()); } PartHandle::PartHandle(DataHandle& handle, const OffsetList& offset, const LengthList& length) : HandleHolder(handle), pos_(0), index_(0), offset_(offset), length_(length) { ASSERT(offset_.size() == length_.size()); } PartHandle::PartHandle(DataHandle* handle, const OffsetList& offset, const LengthList& length) : HandleHolder(handle), pos_(0), index_(0), offset_(offset), length_(length) { ASSERT(offset_.size() == length_.size()); } PartHandle::PartHandle(DataHandle& handle, const Offset& offset, const Length& length) : HandleHolder(handle), pos_(0), index_(0), offset_(1, offset), length_(1, length) { ASSERT(offset_.size() == length_.size()); } PartHandle::PartHandle(DataHandle* handle, const Offset& offset, const Length& length) : HandleHolder(handle), pos_(0), index_(0), offset_(1, offset), length_(1, length) { ASSERT(offset_.size() == length_.size()); } PartHandle::~PartHandle() {} Length PartHandle::openForRead() { handle().openForRead(); rewind(); return estimate(); } void PartHandle::openForWrite(const Length&) { NOTIMP; } void PartHandle::openForAppend(const Length&) { NOTIMP; } long PartHandle::read1(char* buffer, long length) { // skip empty entries if any while (index_ < offset_.size() && length_[index_] == Length(0)) { index_++; } ASSERT(index_ <= offset_.size()); if (index_ == offset_.size()) { return 0; } Length ll = (long long)offset_[index_] + Length(pos_); off_t pos = ll; ASSERT(handle().seek(pos) == Offset(pos)); ll = length_[index_] - Length(pos_); Length lsize = std::min(Length(length), ll); long size = lsize; ASSERT(Length(size) == lsize); long n = handle().read(buffer, size); if (n != size) { std::ostringstream s; s << handle() << ": cannot read " << size << ", got only " << n; throw ReadError(s.str()); } pos_ += n; if (pos_ >= length_[index_]) { index_++; pos_ = 0; } return n; } long PartHandle::read(void* buffer, long length) { char* p = static_cast(buffer); long n = 0; long total = 0; while (length > 0 && (n = read1(p, length)) > 0) { length -= n; total += n; p += n; } return total > 0 ? total : n; } long PartHandle::write(const void*, long) { return -1; } void PartHandle::close() { handle().close(); } void PartHandle::rewind() { pos_ = 0; index_ = 0; } void PartHandle::restartReadFrom(const Offset& from) { Log::warning() << *this << " restart read from " << from << std::endl; rewind(); long long len = from; long long pos = 0; for (index_ = 0; index_ < length_.size(); index_++) { long long e = length_[index_]; if (len >= pos && len < pos + e) { Log::warning() << *this << " restart read from " << from << ", index=" << index_ << ", pos=" << pos_ << std::endl; pos_ = len - pos; return; } pos += e; } ASSERT(from == Offset(0) && estimate() == Length(0)); } Length PartHandle::estimate() { return std::accumulate(length_.begin(), length_.end(), Length(0)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/SharedHandle.cc0000664000175000017500000000703315161702250020133 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit.h" #include "eckit/config/Resource.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/io/SharedHandle.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- SharedHandle::SharedHandle(DataHandle& handle) : handle_(handle) {} SharedHandle::~SharedHandle() {} void SharedHandle::print(std::ostream& s) const { if (format(s) == Log::compactFormat) { s << "SharedHandle"; } else { s << "SharedHandle[handle=" << handle_ << ']'; } } Length SharedHandle::openForRead() { // Just ignore, assumes handle already opened rewind(); return estimate(); } void SharedHandle::openForWrite(const Length&) { // Just ignore, assumes handle already opened rewind(); } void SharedHandle::openForAppend(const Length&) { // Just ignore, assumes handle already opened rewind(); } long SharedHandle::read(void* data, long len) { return handle_.read(data, len); } long SharedHandle::write(const void* data, long len) { return handle_.write(data, len); } void SharedHandle::close() { // Just ignore, assumes handle closed somewhere else } void SharedHandle::flush() { return handle_.flush(); } Length SharedHandle::size() { return handle_.size(); } Length SharedHandle::estimate() { return handle_.estimate(); } Offset SharedHandle::position() { return handle_.position(); } Offset SharedHandle::seek(const Offset& o) { return handle_.seek(o); } bool SharedHandle::canSeek() const { return handle_.canSeek(); } void SharedHandle::skip(const Length& n) { return handle_.skip(n); } void SharedHandle::rewind() { return handle_.rewind(); } void SharedHandle::restartReadFrom(const Offset& o) { return handle_.restartReadFrom(o); } void SharedHandle::restartWriteFrom(const Offset& o) { return handle_.restartWriteFrom(o); } DataHandle* SharedHandle::clone() const { NOTIMP; } std::string SharedHandle::name() const { return handle_.name(); } bool SharedHandle::compress(bool b) { return handle_.compress(b); } bool SharedHandle::merge(DataHandle*) { NOTIMP; } bool SharedHandle::isEmpty() const { return handle_.isEmpty(); } bool SharedHandle::moveable() const { return false; } void SharedHandle::toLocal(Stream& s) const { NOTIMP; } DataHandle* SharedHandle::toLocal() { NOTIMP; } void SharedHandle::toRemote(Stream& s) const { NOTIMP; } void SharedHandle::selectMover(MoverTransferSelection&, bool) const { NOTIMP; } std::string SharedHandle::title() const { return handle_.title(); } void SharedHandle::collectMetrics(const std::string& what) const { handle_.collectMetrics(what); } Length SharedHandle::saveInto(DataHandle& other, TransferWatcher& watcher) { return handle_.saveInto(other, watcher); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/FOpenDataHandle.cc0000664000175000017500000001576115161702250020535 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/eckit.h" #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/io/DblBuffer.h" #include "eckit/io/MoverTransfer.h" #include "eckit/log/Bytes.h" #include "eckit/log/Progress.h" #include "eckit/log/Timer.h" namespace eckit { class FOpenDataHandle { DataHandle* handle_; bool delete_on_close_; bool open_close_; off_t position_; // Keep track of position to cater for public: FOpenDataHandle(DataHandle* handle, const char* mode, bool delete_on_close, bool open_close); ~FOpenDataHandle(); long read(char* buffer, long length); long write(const char* buffer, long length); off_t seek(off_t pos, int whence); int close(); char buffer_[16 * 1024]; }; FOpenDataHandle::FOpenDataHandle(DataHandle* handle, const char* mode, bool delete_on_close, bool open_close) : handle_(handle), delete_on_close_(delete_on_close), open_close_(open_close), position_(0) { if (open_close_) { bool ok = false; if (::strcmp(mode, "r") == 0) { handle_->openForRead(); ok = true; } if (::strcmp(mode, "w") == 0) { handle_->openForWrite(0); ok = true; } if (::strcmp(mode, "a") == 0) { handle_->openForAppend(0); ok = true; } ASSERT(ok); } } FOpenDataHandle::~FOpenDataHandle() { if (open_close_) { handle_->close(); } if (delete_on_close_) { delete handle_; } } long FOpenDataHandle::read(char* buffer, long length) { long len = handle_->read(buffer, length); if (len > 0) { position_ += len; } return len; } long FOpenDataHandle::write(const char* buffer, long length) { long len = handle_->write(buffer, length); if (len > 0) { position_ += len; } return len; } off_t FOpenDataHandle::seek(off_t pos, int whence) { off_t where = pos; switch (whence) { case SEEK_SET: where = pos; break; case SEEK_CUR: where = position_ + pos; break; case SEEK_END: where = (long long)(handle_->size()) - pos; break; default: std::ostringstream oss; oss << "FOpenDataHandle can't seek(pos=" << pos << ", whence=" << whence << ")"; throw NotImplemented(oss.str(), Here()); } if (where == position_) { return where; } off_t w = handle_->seek(where); ASSERT(w == where); position_ = w; return w; } int FOpenDataHandle::close() { delete this; return 0; } static ssize_t readfn(void* data, char* buffer, size_t length) { try { FOpenDataHandle* fd = reinterpret_cast(data); ssize_t len = fd->read(buffer, length); if (len < 0) { // Some datahandles return -1 on EOF len = 0; } return len; } catch (std::exception& e) { // Catch all exceptions on a possible C/C++ boundary eckit::Log::error() << "Exception caught in wrapped DataHandle read: " << e.what() << std::endl; // See man fopencookie. Returns -1 on error return -1; } } static ssize_t writefn(void* data, const char* buffer, size_t length) { try { FOpenDataHandle* fd = reinterpret_cast(data); ssize_t len = fd->write(buffer, length); if (len < 0) { // Some datahandles return -1 on error len = 0; } return len; } catch (std::exception& e) { // Catch all exceptions on a possible C/C++ boundary eckit::Log::error() << "Exception caught in wrapped DataHandle write: " << e.what() << std::endl; // See man fopencookie. Returns 0 on error return 0; } } static int seekfn(void* data, off_t* offset, int whence) { try { FOpenDataHandle* fd = reinterpret_cast(data); *offset = fd->seek(*offset, whence); return 0; } catch (std::exception& e) { // Catch all exceptions on a possible C/C++ boundary eckit::Log::error() << "Exception caught in wrapped DataHandle seek: " << e.what() << std::endl; // See man fopencookie. Returns -1 on error return -1; } } static int closefn(void* data) { try { FOpenDataHandle* fd = reinterpret_cast(data); return fd->close(); } catch (std::exception& e) { // Catch all exceptions on a possible C/C++ boundary eckit::Log::error() << "Exception caught in wrapped DataHandle close: " << e.what() << std::endl; // See man fopencookie. Returns -1 on error return EOF; } } #if eckit_HAVE_FOPENCOOKIE FILE* opencookie(FOpenDataHandle* h, const char* mode) { ASSERT(sizeof(long) >= sizeof(size_t)); ASSERT(sizeof(long) >= sizeof(ssize_t)); cookie_io_functions_t func = {&readfn, &writefn, &seekfn, &closefn}; return ::fopencookie(h, mode, func); } #elif eckit_HAVE_FUNOPEN static int _read(void* data, char* buffer, int length) { return readfn(data, buffer, length); } static int _write(void* data, const char* buffer, int length) { int len = writefn(data, buffer, length); if (len == 0 && length > 0) { return -1; } return len; } static fpos_t _seek(void* data, fpos_t pos, int whence) { if (seekfn(data, &pos, whence) < 0) { return -1; } return pos; } static int _close(void* data) { return closefn(data); } FILE* opencookie(FOpenDataHandle* h, const char* mode) { return ::funopen(h, &_read, &_write, &_seek, &_close); } #else FILE* opencookie(FOpenDataHandle* h, const char* mode) { NOTIMP; } #endif static FILE* open(DataHandle* handle, const char* mode, bool delete_on_close, bool open_close) { FOpenDataHandle* h = new FOpenDataHandle(handle, mode, delete_on_close, open_close); FILE* f = opencookie(h, mode); if (handle->canSeek()) { setvbuf(f, h->buffer_, _IOFBF, sizeof(h->buffer_)); } else { setvbuf(f, nullptr, _IONBF, 0); } return f; } FILE* DataHandle::openf(const char* mode, bool delete_on_close) { return open(this, mode, delete_on_close, true); } FILE* DataHandle::openf(bool delete_on_close) { return open(this, "r+", delete_on_close, false); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/MultiSocketHandle.cc0000664000175000017500000000563215161702250021173 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/io/MultiSocketHandle.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec MultiSocketHandle::classSpec_ = { &DataHandle::classSpec(), "MultiSocketHandle", }; Reanimator MultiSocketHandle::reanimator_; void MultiSocketHandle::print(std::ostream& s) const { s << "MultiSocketHandle[host=" << host_ << ",port=" << port_ << ']'; } void MultiSocketHandle::encode(Stream& s) const { DataHandle::encode(s); s << host_; s << port_; s << streams_; s << messageSize_; s << bufferSize_; } MultiSocketHandle::MultiSocketHandle(Stream& s) : DataHandle(s), port_(0) { s >> host_; s >> port_; s >> streams_; s >> messageSize_; s >> bufferSize_; } MultiSocketHandle::MultiSocketHandle(const std::string& host, int port, size_t streams, size_t messageSize, size_t bufferSize) : host_(host), port_(port), streams_(streams), messageSize_(messageSize), bufferSize_(bufferSize) {} MultiSocketHandle::~MultiSocketHandle() {} Length MultiSocketHandle::openForRead() { connection_.reset(new net::MultiSocket(streams_, messageSize_)); connection_->bufferSize(bufferSize_); connection_->connect(host_, port_); return 0; } void MultiSocketHandle::openForWrite(const Length&) { connection_.reset(new net::MultiSocket(streams_, messageSize_)); connection_->bufferSize(bufferSize_); connection_->connect(host_, port_); } void MultiSocketHandle::openForAppend(const Length&) { NOTIMP; } long MultiSocketHandle::read(void* buffer, long length) { return connection_->read(buffer, length); } long MultiSocketHandle::write(const void* buffer, long length) { return connection_->write(buffer, length); } void MultiSocketHandle::close() { connection_->close(); connection_.reset(nullptr); } void MultiSocketHandle::rewind() { NOTIMP; } DataHandle* MultiSocketHandle::clone() const { return new MultiSocketHandle(host_, port_, streams_, messageSize_); } std::string MultiSocketHandle::title() const { std::ostringstream os; os << "TCP[" << host_ << ":" << port_ << "]"; return os.str(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/FileHandle.h0000664000175000017500000000470115161702250017445 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date May 1996 #ifndef eckit_io_FileHandle_h #define eckit_io_FileHandle_h #include #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" namespace eckit { class FileHandle : public DataHandle { public: FileHandle(const std::string&, bool = false); FileHandle(Stream&); ~FileHandle() override; void advance(const Length&); const std::string& path() const { return name_; } // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; void print(std::ostream&) const override; Length size() override; Length estimate() override; Offset position() override; bool isEmpty() const override; void restartReadFrom(const Offset& from) override; void restartWriteFrom(const Offset& from) override; void toRemote(Stream&) const override; void selectMover(MoverTransferSelection&, bool read) const override; std::string title() const override; std::string metricsTag() const override; bool moveable() const override { return true; } Offset seek(const Offset&) override; bool canSeek() const override; void skip(const Length&) override; DataHandle* clone() const override; void hash(MD5& md5) const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // members std::string name_; bool overwrite_; FILE* file_; bool read_; std::unique_ptr buffer_; private: // methods void open(const char*); static ClassSpec classSpec_; static Reanimator reanimator_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/MMappedFileHandle.h0000664000175000017500000000436015161702250020712 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date May 2020 #ifndef eckit_io_MMappedFileHandle_h #define eckit_io_MMappedFileHandle_h #include #include "eckit/io/DataHandle.h" namespace eckit { class MMappedFileHandle : public DataHandle { public: MMappedFileHandle(const std::string&); MMappedFileHandle(Stream&); ~MMappedFileHandle() override; const std::string& path() const { return path_; } // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; void print(std::ostream&) const override; Length size() override; Length estimate() override; Offset position() override; bool isEmpty() const override; void restartReadFrom(const Offset& from) override; void restartWriteFrom(const Offset& from) override; std::string title() const override; std::string metricsTag() const override; Offset seek(const Offset&) override; bool canSeek() const override { return true; } void skip(const Length&) override; DataHandle* clone() const override; void hash(MD5& md5) const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // members std::string path_; std::unique_ptr handle_; void* mmap_; int fd_; off_t length_; private: // methods void open(const char*); static ClassSpec classSpec_; static Reanimator reanimator_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/TransferWatcher.h0000664000175000017500000000220015161702250020544 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File TransferWatcher.h // Baudouin Raoult - ECMWF Jun 98 #ifndef eckit_TransferWatcher_h #define eckit_TransferWatcher_h //----------------------------------------------------------------------------- namespace eckit { class Offset; //----------------------------------------------------------------------------- class TransferWatcher { public: // -- Methods virtual void watch(const void*, long) = 0; virtual void restartFrom(const Offset&) {} virtual void fromHandleOpened() {} virtual void toHandleOpened() {} virtual ~TransferWatcher() {} // -- Class methods static TransferWatcher& dummy(); }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/Base64.cc0000664000175000017500000000531315161702250016634 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/Base64.h" #include "eckit/io/BitIO.h" #include "eckit/io/DataHandle.h" // This code is written for readibility, not speed namespace eckit { static const char* codes_b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789+/"; static const char* codes_url = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789-_"; Base64::Base64(bool url) { const char* p = url ? codes_url : codes_b64; size_t i = 0; while (*p) { size_t j = *p; encode_[i] = *p; decode_[j] = i; p++; i++; } } size_t Base64::encode(DataHandle& in, DataHandle& out) { const size_t EOI_MARKER = 256; BitIO bin(in, true); BitIO bout(out); size_t c; while ((c = bin.read(6, EOI_MARKER)) != EOI_MARKER) { bout.write(encode_[c], 8); } switch (bin.bitCount() % 6) { case 0: // No padding break; case 2: bout.write('=', 8); bout.write('=', 8); break; case 4: bout.write('=', 8); break; default: { std::ostringstream oss; oss << "Base64: invalid padding: " << (bin.bitCount() % 6); throw eckit::SeriousBug(oss.str()); } } return bout.byteCount(); } //---------------------------------------------------------------------------------------------------------------------- size_t Base64::decode(DataHandle& in, DataHandle& out) { const size_t EOI_MARKER = 256; BitIO bin(in); BitIO bout(out); size_t c = bin.read(8, EOI_MARKER); size_t prev = EOI_MARKER; while (c != EOI_MARKER) { if (c == '=') { size_t left = 8 - (bout.bitCount() % 8); bout.write(decode_[prev] >> (6 - left), left); prev = EOI_MARKER; break; } if (prev != EOI_MARKER) { bout.write(decode_[prev], 6); } prev = c; c = bin.read(8, EOI_MARKER); } if (prev != EOI_MARKER) { bout.write(decode_[prev], 6); } return bout.byteCount(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/Buffer.h0000664000175000017500000000625115161702250016665 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date July 1996 #ifndef eckit_io_Buffer_h #define eckit_io_Buffer_h #include #include namespace eckit { /// Simple class to implement memory buffers class Buffer { public: // methods /// Creates a buffer with 'size' many bytes. /// @param size in bytes of the buffer. explicit Buffer(size_t size = 0); /// Creates a buffer from the string s. /// NOTE: the buffer will contain the string as a zero terminated c-string. /// I.e. the resulting size of buffer is s.size() + 1 /// @param s, initial content of buffer, including null-terminator explicit Buffer(const std::string& s); /// Creates buffer with initial content /// @param src to copy bytes from /// @param size of data Buffer(const void* src, size_t size); /// Copy constructor (deleted). Buffer(const Buffer&) = delete; /// Copy assignment (deleted). Buffer& operator=(const Buffer&) = delete; /// Move constructor. Note that rhs is not guaranteed to be valid! Buffer(Buffer&& rhs) noexcept; /// Move assignment. Note that rhs is not guaranteed to be valid! Buffer& operator=(Buffer&& rhs) noexcept; ~Buffer(); operator char*() { return buffer_; } operator const char*() const { return buffer_; } operator void*() { return buffer_; } operator const void*() const { return buffer_; } void* data() { return buffer_; } const void* data() const { return buffer_; } /// Returns size of the buffer in bytes /// Note: The actual allocation held by this buffer may be larger /// if the buffer has been resized with 'preserveData' set to a smaller size than before. /// @return buffer size in bytes size_t size() const { return size_; } /// Zero content of buffer void zero(); /// Resizes the buffer to newSize. /// When 'preserveData' is set: /// - Buffer will still hold the data contained before resize() was called. /// - If 'newSize' is smaller than the current 'size' the data will be truncated. /// With unset 'preserveData' the contents of the buffer will be discarded. void resize(size_t newSize, bool preserveData = false); /// Copy data of given size (bytes) into buffer at given position /// @pre Buffer must have sufficient size void copy(const void*, size_t size, size_t pos = 0); protected: // methods void create(); void destroy(); /// Copies contents of s into the buffer including a null-terminator at the end. /// This copy operation truncates s if the buffer is not big enough. /// @param s, string to be copied including null-terminator. void copy(const std::string& s); private: // members char* buffer_{nullptr}; size_t size_{0}; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/PooledHandle.h0000664000175000017500000000272415161702250020013 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 2019 #ifndef eckit_io_PooledHandle_h #define eckit_io_PooledHandle_h #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" namespace eckit { class PoolHandleEntry; class PooledHandle : public DataHandle { public: PooledHandle(const PathName& name); /// @pre must have been closed ~PooledHandle(); Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; Offset seek(const Offset&) override; bool canSeek() const override { return true; } void hash(MD5& md5) const override; Offset position() override; // for testing size_t nbOpens() const; size_t nbReads() const; size_t nbSeeks() const; private: PathName path_; PoolHandleEntry* entry_; void print(std::ostream& s) const override; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/PipeHandle.h0000664000175000017500000000447515161702250017473 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/PipeHandle.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_filesystem_PipeHandle_h #define eckit_filesystem_PipeHandle_h #include "eckit/io/DataHandle.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class PipeHandle : public DataHandle { public: // -- Contructors PipeHandle(const std::string&); PipeHandle(Stream&); // -- Destructor ~PipeHandle(); // -- Methods void advance(const Length&); // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void rewind() override; void print(std::ostream&) const override; /* virtual void restartReadFrom(const Offset& from); virtual void restartWriteFrom(const Offset& from); */ /* virtual void toRemote(Stream&) const; */ /* virtual void cost(std::map&, bool) const; */ /* std::string title() const override; */ bool moveable() const override { return false; } Offset seek(const Offset&) override; bool canSeek() const override { return false; } // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- Members std::string name_; FILE* file_; bool read_; // -- Methods void open(const char*); // -- Class members static ClassSpec classSpec_; static Reanimator reanimator_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/StdFile.cc0000664000175000017500000000202715161702250017141 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/StdFile.h" namespace eckit { StdFile::StdFile(const PathName& name, const std::string& mode) : file_{nullptr} { file_ = ::fopen(name.localPath(), mode.c_str()); if (file_ == nullptr) { throw CantOpenFile(name); } } StdFile::~StdFile() { ASSERT_MSG(!isOpen(), "StdFile hasn't been closed before destruction"); } void StdFile::close() noexcept(false) { if (isOpen()) { if (fclose(file_)) { throw FailedSystemCall("fclose"); } } file_ = nullptr; } } // namespace eckit eckit-2.0.7/src/eckit/io/TeeHandle.h0000664000175000017500000000445615161702250017312 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/TeeHandle.h // Manuel Fuentes - ECMWF Jul 96 #ifndef eckit_filesystem_TeeHandle_h #define eckit_filesystem_TeeHandle_h #include "eckit/io/DataHandle.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class TeeHandle : public DataHandle { public: using HandleList = std::vector; // -- Contructors TeeHandle(); TeeHandle(DataHandle*, DataHandle*); TeeHandle(const HandleList&); TeeHandle(Stream&); // -- Destructor ~TeeHandle(); // -- Operators virtual void operator+=(DataHandle*); // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; void print(std::ostream&) const override; bool canSeek() const override { return false; } void toRemote(Stream&) const override; void toLocal(Stream&) const override; DataHandle* toLocal() override; void selectMover(MoverTransferSelection&, bool read) const override; bool moveable() const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- Members HandleList datahandles_; // -- Methods // -- Class members static ClassSpec classSpec_; static Reanimator reanimator_; mutable std::set requiredAttributes_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/PooledFileDescriptor.h0000664000175000017500000000214315161702250021531 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Olivier Iffrig /// @date Oct 2019 #ifndef eckit_io_PooledFileDescriptor_h #define eckit_io_PooledFileDescriptor_h #include #include "eckit/filesystem/PathName.h" #include "eckit/io/PooledFile.h" namespace eckit { class PooledFileDescriptor { public: PooledFileDescriptor(const PathName& path, bool readOnly); ~PooledFileDescriptor(); void open(); void close(); ssize_t read(void*, size_t); ssize_t write(const void*, size_t); void sync(); off_t seek(off_t offset); off_t seekEnd(); int fileno() const { return fd_; } private: PathName path_; std::unique_ptr file_; int fd_; bool readOnly_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/FileLock.h0000664000175000017500000000260315161702250017141 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// /// @date Oct 2016 #ifndef eckit_io_FileLock_h #define eckit_io_FileLock_h #include "eckit/filesystem/PathName.h" #include "eckit/io/FileLocker.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class FileLock { public: /// Constructor /// creates the lock file if needed explicit FileLock(const PathName& lockFile, bool unlink_at_destruction = false); FileLock(const FileLock&) = delete; FileLock& operator=(const FileLock&) = delete; FileLock(FileLock&&) = delete; FileLock& operator=(FileLock&&) = delete; ~FileLock(); void lock(); void unlock(); private: const PathName lockFile_; int fd_; FileLocker locker_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/DblBuffer.h0000664000175000017500000000314715161702250017310 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Feb 97 #ifndef eckit_DblBuffer_h #define eckit_DblBuffer_h #include "eckit/io/DataHandle.h" #include "eckit/io/Length.h" #include "eckit/io/TransferWatcher.h" #include "eckit/thread/Mutex.h" namespace eckit { class DblBuffer { public: // -- Contructors DblBuffer(long count = 5, long size = 1024 * 1024, TransferWatcher& = TransferWatcher::dummy()); DblBuffer(const DblBuffer&) = delete; DblBuffer& operator=(const DblBuffer&) = delete; DblBuffer(DblBuffer&&) = delete; DblBuffer& operator=(DblBuffer&&) = delete; // -- Destructor ~DblBuffer(); // -- Methods Length copy(DataHandle&, DataHandle&); bool error(); void error(const std::string&); void restart(RestartTransfer&); private: // methods Length copy(DataHandle&, DataHandle&, const Length&); private: // members Mutex mutex_; long count_; long bufSize_; Length inBytes_; Length outBytes_; bool error_; std::string why_; bool restart_; Offset restartFrom_; TransferWatcher& watcher_; // -- Friends friend class DblBufferTask; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/BufferCache.cc0000664000175000017500000000410415161702250017742 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File BufferCache.cc // Baudouin Raoult - (c) ECMWF Jul 11 #include "eckit/io/BufferCache.h" #include "eckit/serialisation/Stream.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- BufferCache::BufferCache(size_t size) : count_(0), buffer_(size), updated_(::time(nullptr)) {} BufferCache::BufferCache(const BufferCache& other) : count_(other.count_), buffer_(other.buffer_.size()), updated_(::time(nullptr)) { ::memcpy((char*)buffer_, (const char*)other.buffer_, count_); } BufferCache::~BufferCache() {} BufferCache& BufferCache::operator=(const BufferCache& other) { if (this != &other) { count_ = other.count_; buffer_.resize(other.buffer_.size()); ::memcpy((char*)buffer_, (const char*)other.buffer_, count_); updated_ = ::time(nullptr); } return *this; } bool BufferCache::operator<(const BufferCache& other) const { return (count_ < other.count_) || ((count_ == other.count_) && (::memcmp(buffer_, other.buffer_, count_) < 0)); } void BufferCache::reset() { count_ = 0; } void BufferCache::add(const void* buffer, size_t len) { if (buffer_.size() < count_ + len) { buffer_.resize(count_ + len + 1024); } ::memcpy(((char*)buffer_) + count_, buffer, len); count_ += len; } void BufferCache::print(std::ostream& s) const { Stream::dump(s, buffer_, count_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/PipeHandle.cc0000664000175000017500000000503015161702250017615 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/PipeHandle.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec PipeHandle::classSpec_ = { &DataHandle::classSpec(), "PipeHandle", }; Reanimator PipeHandle::reanimator_; void PipeHandle::print(std::ostream& s) const { s << "PipeHandle[file=" << name_ << ']'; } void PipeHandle::encode(Stream& s) const { DataHandle::encode(s); s << name_; } PipeHandle::PipeHandle(Stream& s) : DataHandle(s), file_{nullptr}, read_(false) { s >> name_; } PipeHandle::PipeHandle(const std::string& name) : name_(name), file_{nullptr}, read_(false) {} PipeHandle::~PipeHandle() {} void PipeHandle::open(const char* mode) { file_ = ::popen(name_.c_str(), mode); if (file_ == nullptr) { throw CantOpenFile(name_); } } Length PipeHandle::openForRead() { read_ = true; open("r"); return estimate(); } void PipeHandle::openForWrite(const Length&) { open("w"); } void PipeHandle::openForAppend(const Length&) { open("a"); } long PipeHandle::read(void* buffer, long length) { return ::fread(buffer, 1, length, file_); } long PipeHandle::write(const void* buffer, long length) { return ::fwrite(buffer, 1, length, file_); } void PipeHandle::close() { if (file_ == nullptr) { return; } if (::pclose(file_) != 0) { Log::error() << "pclose(" << name_ << ')' << Log::syserr << std::endl; throw WriteError(name()); } } void PipeHandle::rewind() { NOTIMP; } void PipeHandle::advance(const Length& len) { NOTIMP; } /* void PipeHandle::restartReadFrom(const Offset& from) { NOTIMP; } void PipeHandle::restartWriteFrom(const Offset& from) { NOTIMP; } */ Offset PipeHandle::seek(const Offset& from) { NOTIMP; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/MemoryHandle.cc0000664000175000017500000001167115161702250020200 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/io/MemoryHandle.h" #include "eckit/maths/Functions.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- MemoryHandle::MemoryHandle(const Buffer& buffer) : address_(const_cast(buffer)), size_(buffer.size()), capacity_(buffer.size()), opened_(false), readOnly_(true), read_(false), grow_(false), owned_(false), position_(0) {} MemoryHandle::MemoryHandle(Buffer& buffer) : address_(buffer), size_(buffer.size()), capacity_(buffer.size()), opened_(false), readOnly_(false), read_(false), grow_(false), owned_(false), position_(0) {} MemoryHandle::MemoryHandle(const void* address, size_t size) : address_(const_cast(reinterpret_cast(address))), size_(size), capacity_(size), opened_(false), readOnly_(true), read_(false), grow_(false), owned_(false), position_(0) {} MemoryHandle::MemoryHandle(void* address, size_t size) : address_(reinterpret_cast(address)), size_(size), capacity_(size), opened_(false), readOnly_(false), read_(false), grow_(false), owned_(false), position_(0) {} MemoryHandle::MemoryHandle(size_t size, bool grow) : address_{nullptr}, size_(0), capacity_(size), opened_(false), readOnly_(false), read_(false), grow_(grow), owned_(true), position_(0) { address_ = new char[capacity_]; ASSERT(address_); } MemoryHandle::~MemoryHandle() { if (owned_) { delete[] address_; } } Length MemoryHandle::openForRead() { read_ = true; position_ = 0; opened_ = true; return size_; } void MemoryHandle::openForWrite(const Length&) { ASSERT(!readOnly_); read_ = false; position_ = 0; opened_ = true; } void MemoryHandle::openForAppend(const Length&) { NOTIMP; } void MemoryHandle::skip(const Length& len) { ASSERT(read_); seek(position() + len); } long MemoryHandle::read(void* buffer, long length) { ASSERT(opened_); ASSERT(read_); ASSERT(length >= 0); size_t left = size_ - position_; size_t size = std::min(left, size_t(length)); ::memcpy(buffer, address_ + position_, size); position_ += size; return size; } long MemoryHandle::write(const void* buffer, long length) { ASSERT(opened_); ASSERT(!read_); ASSERT(length >= 0); size_t left = size_ - position_; if (grow_ && (left < size_t(length))) { if ((capacity_ - position_) < size_t(length)) { size_t newcapacity = round(capacity_ + size_t(length), 1024 * 1024); char* newdata = new char[newcapacity]; ASSERT(newdata); ::memcpy(newdata, address_, position_); delete[] address_; address_ = newdata; capacity_ = newcapacity; } size_ += length; left = size_ - position_; } size_t size = std::min(left, size_t(length)); ::memcpy(address_ + position_, buffer, size); position_ += size; return size; } void MemoryHandle::close() { opened_ = false; } void MemoryHandle::flush() {} void MemoryHandle::rewind() { ASSERT(opened_); position_ = 0; } Offset MemoryHandle::seek(const Offset& off) { ASSERT(opened_); ASSERT(size_t(off) <= size_); position_ = off; return position_; } void MemoryHandle::print(std::ostream& s) const { s << "MemoryHandle[size=" << size_ << ']'; } Length MemoryHandle::estimate() { return size_; } Offset MemoryHandle::position() { ASSERT(opened_); return position_; } std::string MemoryHandle::title() const { return ""; } DataHandle* MemoryHandle::clone() const { if (owned_) { MemoryHandle* h = new MemoryHandle(size_, grow_); ::memcpy(h->address_, address_, size_); h->size_ = size_; return h; } if (readOnly_) { return new MemoryHandle(static_cast(address_), size_); } return new MemoryHandle(address_, size_); } const void* MemoryHandle::data() const { return address_; } Length MemoryHandle::size() { return size_; } std::string MemoryHandle::str() const { return std::string(address_, address_ + position_); } Length MemoryHandle::size() const { return size_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/RawFileHandle.cc0000664000175000017500000000530115161702250020252 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/RawFileHandle.h" #include "eckit/os/Stat.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- void RawFileHandle::print(std::ostream& s) const { s << "RawFileHandle[" << path_ << ']'; } void RawFileHandle::encode(Stream& s) const { NOTIMP; } RawFileHandle::RawFileHandle(const std::string& path, bool overwrite) : path_(path), overwrite_(overwrite), fd_(-1) {} RawFileHandle::~RawFileHandle() { if (fd_ != -1) { close(); } } Length RawFileHandle::openForRead() { SYSCALL(fd_ = ::open(std::string(path_).c_str(), O_RDONLY)); SYSCALL(::fcntl(fd_, F_SETFD, FD_CLOEXEC)); struct stat st; SYSCALL(::fstat(fd_, &st)); ASSERT(sizeof(st.st_size) == sizeof(long long)); return st.st_size; } void RawFileHandle::openForWrite(const Length&) { if (overwrite_) { fd_ = SYSCALL(::open(std::string(path_).c_str(), O_WRONLY, 0777)); } else { fd_ = SYSCALL(::open(std::string(path_).c_str(), O_WRONLY | O_CREAT, 0777)); } SYSCALL(::fcntl(fd_, F_SETFD, FD_CLOEXEC)); } void RawFileHandle::openForAppend(const Length&) { NOTIMP; } long RawFileHandle::read(void* buffer, long length) { long n; SYSCALL(n = ::read(fd_, buffer, length)); return n; } long RawFileHandle::write(const void* buffer, long length) { long n; SYSCALL(n = ::write(fd_, buffer, length)); return n; } void RawFileHandle::close() { /// @todo fsync SYSCALL(::close(fd_)); fd_ = -1; } Offset RawFileHandle::position() { return ::lseek(fd_, 0, SEEK_CUR); } Offset RawFileHandle::seek(const Offset& o) { return ::lseek(fd_, o, SEEK_SET); } void RawFileHandle::skip(const Length& l) { ::lseek(fd_, l, SEEK_CUR); } Length RawFileHandle::size() { Stat::Struct info; SYSCALL(Stat::fstat(fd_, &info)); return info.st_size; } Length RawFileHandle::estimate() { Stat::Struct info; SYSCALL(Stat::fstat(fd_, &info)); return info.st_size; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/Length.h0000664000175000017500000000442215161702250016673 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Length.h // Baudouin Raoult - ECMWF Jul 96 #ifndef eckit_Length_h #define eckit_Length_h #include #include #include "eckit/persist/Bless.h" namespace eckit { //----------------------------------------------------------------------------- class Bless; class DumpLoad; class Stream; // But because the compiler aligns long longs // on 64bits boundaries and longs on 32 bits boundaries, // we need the help of a little pragma here, to make ObjectStore happy #ifdef _AIX #pragma options align = twobyte #endif class Length { public: // types using value_t = long long; public: // methods friend std::ostream& operator<<(std::ostream& s, const Length& x); friend Stream& operator<<(Stream& s, const Length& x); friend Stream& operator>>(Stream& s, Length& x); // -- Contructors Length(long long l = 0) : value_(l) {} Length(const Length& other) : value_(other.value_) {} #include "eckit/io/Length.b" public: // operators Length& operator=(const Length& other) { value_ = other.value_; return *this; } Length operator+(const Length& other) const { return Length(value_ + other.value_); } Length& operator+=(const Length& other) { value_ += other.value_; return *this; } bool operator==(const Length& other) const { return value_ == other.value_; } bool operator<(const Length& other) const { return value_ < other.value_; } operator long long() const { return value_; } long long operator-(const Length& other) const { return value_ - other.value_; } void dump(DumpLoad&) const; void load(DumpLoad&); private: // members value_t value_; friend class Offset; }; using LengthList = std::vector; #ifdef _AIX #pragma options align = reset #endif //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/rados/0000775000175000017500000000000015161702250016407 5ustar alastairalastaireckit-2.0.7/src/eckit/io/rados/RadosObject.cc0000664000175000017500000000341315161702250021116 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/rados/RadosObject.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/Tokenizer.h" namespace eckit { RadosObject::RadosObject(Stream& s) { s >> pool_; s >> oid_; } void RadosObject::encode(Stream& s) const { s << pool_; s << oid_; } RadosObject::RadosObject(const RadosObject& other, size_t part) { *this = other; if (part) { std::ostringstream oss; oss << oid_ << ";part-" << part; oid_ = oss.str(); } } RadosObject::RadosObject(const std::string& path) { // static const std::string defaultRadosPool = Resource("defaultRadosPool", "default"); static const std::string defaultRadosPool = Resource("defaultRadosPool", "cephfs_data"); Tokenizer parse(":"); std::vector bits; parse(path, bits); ASSERT(bits.size() == 1 || bits.size() == 2); if (bits.size() == 1) { oid_ = path; pool_ = defaultRadosPool; } else { pool_ = bits[0]; oid_ = bits[1]; } } RadosObject::RadosObject(const std::string& pool, const std::string& oid) : pool_(pool), oid_(oid) {} std::string RadosObject::str() const { return pool_ + ':' + oid_; } void RadosObject::print(std::ostream& s) const { s << "RadosObject[pool=" << pool_ << ",oid=" << oid_ << "]"; } } // namespace eckit eckit-2.0.7/src/eckit/io/rados/RadosCluster.cc0000664000175000017500000001550715161702250021340 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/rados/RadosCluster.h" #include "eckit/io/rados/RadosAttributes.h" #include "eckit/io/rados/RadosObject.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" namespace eckit { class RadosIOCtx { public: rados_ioctx_t io_; RadosIOCtx(rados_t cluster, const std::string& pool) { std::cout << "RadosIOCtx => rados_ioctx_create(" << pool << ")" << std::endl; RADOS_CALL(rados_ioctx_create(cluster, pool.c_str(), &io_)); std::cout << "RadosIOCtx <= rados_ioctx_create(" << pool << ")" << std::endl; } ~RadosIOCtx() { std::cout << "~RadosIOCtx => rados_ioctx_destroy(io_ctx_)" << std::endl; rados_ioctx_destroy(io_); std::cout << "~RadosIOCtx <= rados_ioctx_destroy(io_ctx_)" << std::endl; } }; const RadosCluster& RadosCluster::instance() { thread_local RadosCluster instance_; return instance_; } RadosCluster::RadosCluster() : cluster_(0) { static const std::string radosClusterName = Resource("radosClusterName", "mars"); static const std::string radosClusterUser = Resource("radosClusterUser", "client.mars"); static const std::string radosClusterConf = Resource("radosClusterConf", "~/.ceph/ceph.conf"); static const PathName radosClusterConfPath(radosClusterConf, true); uint64_t flags = 0; std::cout << "RadosClusterName is " << radosClusterName << std::endl; std::cout << "RadosClusterUser is " << radosClusterUser << std::endl; /* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */ RADOS_CALL(rados_create2(&cluster_, radosClusterName.c_str(), radosClusterUser.c_str(), flags)); std::cout << "RadosClusterConf is " << radosClusterConf << std::endl; RADOS_CALL(rados_conf_read_file(cluster_, radosClusterConfPath.fullName().path().c_str())); RADOS_CALL(rados_connect(cluster_)); } RadosCluster::~RadosCluster() { std::cout << "RadosCluster::~RadosCluster" << std::endl; for (auto j = ctx_.begin(); j != ctx_.end(); ++j) { delete (*j).second; } ctx_.clear(); std::cout << "RADOS_CALL => rados_shutdown(cluster_)" << std::endl; rados_shutdown(cluster_); std::cout << "RADOS_CALL <= rados_shutdown(cluster_)" << std::endl; } void RadosCluster::error(int code, const char* msg, const char* file, int line, const char* func) { std::ostringstream oss; oss << "RADOS error " << msg << ", file " << file << ", line " << line << ", function " << func << " [" << code << "] (" << strerror(-code) << ")"; throw SeriousBug(oss.str()); } Length RadosCluster::maxObjectSize() const { /* rados_ioctx_t& ctx=ioCtx("mars"); char keys[1000000]; size_t keyLen; char vals[1000000]; size_t valLen; rados_application_metadata_list(ctx, "ceph", keys, &keyLen, vals, &valLen); std::cout << vals <("radosMaxObjectSize", 90 * 1024 * 1024); return len; } rados_ioctx_t& RadosCluster::ioCtx(const std::string& pool) const { auto j = ctx_.find(pool); if (j == ctx_.end()) { ctx_[pool] = new RadosIOCtx(cluster_, pool); j = ctx_.find(pool); } return (*j).second->io_; } rados_ioctx_t& RadosCluster::ioCtx(const RadosObject& object) const { return ioCtx(object.pool()); } void RadosCluster::ensurePool(const std::string& pool) const { int64_t id = rados_pool_lookup(cluster_, pool.c_str()); if (id == -ENOENT) { RADOS_CALL(rados_pool_create(cluster_, pool.c_str())); } } void RadosCluster::ensurePool(const RadosObject& object) const { ensurePool(object.pool()); } void RadosCluster::attributes(const RadosObject& object, const RadosAttributes& attr) const { const char* oid = object.oid().c_str(); auto a = attr.attrs(); for (auto j = a.begin(); j != a.end(); ++j) { std::cout << "RadosCluster::attributes => [" << (*j).first << "] [" << (*j).second << "]"; RADOS_CALL(rados_setxattr(ioCtx(object), oid, (*j).first.c_str(), (*j).second.c_str(), (*j).second.size())); } } RadosAttributes RadosCluster::attributes(const RadosObject& object) const { RadosAttributes attr; rados_xattrs_iter_t iter; RADOS_CALL(rados_getxattrs(ioCtx(object), object.oid().c_str(), &iter)); for (;;) { const char* name; const char* val; size_t len; RADOS_CALL(rados_getxattrs_next(iter, &name, &val, &len)); if (!name) { break; } std::cout << "RadosCluster::attributes <= [" << name << "] ["; for (size_t i = 0; i < len; i++) { if (isprint(val[i])) { std::cout << val[i]; } else { std::cout << '.'; } } std::cout << ']' << std::endl; attr.set(name, std::string(val, val + len)); } rados_getxattrs_end(iter); return attr; } void RadosCluster::remove(const RadosObject& object) const { RADOS_CALL(rados_remove(ioCtx(object), object.oid().c_str())); } void RadosCluster::truncate(const RadosObject& object, const Length& length) const { RADOS_CALL(rados_trunc(ioCtx(object), object.oid().c_str(), length)); } bool RadosCluster::exists(const RadosObject& object) const { uint64_t psize; time_t pmtime; int err = rados_stat(ioCtx(object), object.oid().c_str(), &psize, &pmtime); if (err == 0) { return true; } if (err == -ENOENT) { return false; } RADOS_CALL(rados_stat(ioCtx(object), object.oid().c_str(), &psize, &pmtime)); NOTIMP; } Length RadosCluster::size(const RadosObject& object) const { uint64_t psize; time_t pmtime; RADOS_CALL(rados_stat(ioCtx(object), object.oid().c_str(), &psize, &pmtime)); return psize; } time_t RadosCluster::lastModified(const RadosObject& object) const { uint64_t psize; time_t pmtime; RADOS_CALL(rados_stat(ioCtx(object), object.oid().c_str(), &psize, &pmtime)); return pmtime; } void RadosCluster::removeAll(const RadosObject& object) const { RadosAttributes attr = attributes(object); size_t parts; ASSERT(attr.get("parts", parts)); for (size_t i = 0; i < parts; ++i) { remove(RadosObject(object, i)); } } } // namespace eckit eckit-2.0.7/src/eckit/io/rados/README0000664000175000017500000000066715161702250017300 0ustar alastairalastairOn linux: ========= cd /usr/local/apps/ceph/14.2.1-001 tar cf ~/rados-include.tar include/ On mac: ======= brew tap zeichenanonym/ceph-client brew install ceph-client cd /usr/local/Cellar/ceph-client/mimic-13.2.2 tar xf ~/rados-include.tar cmake: ====== cmake -DENABLE_RADOS=ON -DRADOS_PATH=/usr/local/Cellar/ceph-client/mimic-13.2.2 ~/git/eckit/ cmake -DENABLE_RADOS=ON -DRADOS_PATH=/usr/local/apps/ceph/14.2.1-001 ~/git/eckit/ eckit-2.0.7/src/eckit/io/rados/RadosObject.h0000664000175000017500000000253315161702250020762 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 2019 #ifndef eckit_io_rados_RadosObject_h #define eckit_io_rados_RadosObject_h #include #include namespace eckit { class Stream; class RadosObject { public: RadosObject(Stream&); RadosObject(const std::string& path); RadosObject(const std::string& pool, const std::string& oid); RadosObject(const RadosObject& other, size_t part); const std::string& pool() const { return pool_; } const std::string& oid() const { return oid_; } std::string str() const; private: std::string pool_; std::string oid_; void print(std::ostream&) const; void encode(Stream&) const; friend std::ostream& operator<<(std::ostream& s, const RadosObject& o) { o.print(s); return s; } friend Stream& operator<<(Stream& s, const RadosObject& o) { o.encode(s); return s; } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/rados/RadosAttributes.h0000664000175000017500000000411615161702250021701 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 2019 #ifndef eckit_io_rados_RadosAttributes_h #define eckit_io_rados_RadosAttributes_h #include #include #include #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "eckit/utils/Translator.h" namespace eckit { class Stream; class RadosAttributes { public: template void set(const std::string& name, T value) { attrs_[name] = Translator()(value); } void set(const std::string& name, const Length& value) { set(name, static_cast(value)); } void set(const std::string& name, const Offset& value) { set(name, static_cast(value)); } template bool get(const std::string& name, T& value) const { auto j = attrs_.find(name); if (j != attrs_.end()) { value = Translator()((*j).second); return true; } return false; } bool get(const std::string& name, Length& value) { long long v; if (!get(name, v)) { return false; } value = Length(v); return true; } bool get(const std::string& name, Offset& value) { long long v; if (!get(name, v)) { return false; } value = Offset(v); return true; } const std::map& attrs() const { return attrs_; } private: std::map attrs_; void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const RadosAttributes& o) { o.print(s); return s; } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/rados/RadosAttributes.cc0000664000175000017500000000161115161702250022034 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/rados/RadosAttributes.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" namespace eckit { void RadosAttributes::print(std::ostream& s) const { s << "RadosAttributes["; const char* sep = ""; for (auto j = attrs_.begin(); j != attrs_.end(); ++j) { s << sep; s << (*j).first << "=" << (*j).second; sep = ","; } s << "]"; } } // namespace eckit eckit-2.0.7/src/eckit/io/rados/RadosWriteHandle.cc0000664000175000017500000000734715161702250022130 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/rados/RadosAttributes.h" #include "eckit/io/rados/RadosCluster.h" #include "eckit/io/rados/RadosHandle.h" #include "eckit/io/rados/RadosWriteHandle.h" namespace eckit { ClassSpec RadosWriteHandle::classSpec_ = { &DataHandle::classSpec(), "RadosWriteHandle", }; Reanimator RadosWriteHandle::reanimator_; void RadosWriteHandle::print(std::ostream& s) const { s << "RadosWriteHandle[" << object_ << ']'; } void RadosWriteHandle::encode(Stream& s) const { DataHandle::encode(s); s << object_; s << Length(0); // For future extensio } RadosWriteHandle::RadosWriteHandle(Stream& s) : DataHandle(s), object_(s), opened_(false) { s >> maxObjectSize_; if (!maxObjectSize_) { maxObjectSize_ = RadosCluster::instance().maxObjectSize(); } } RadosWriteHandle::RadosWriteHandle(const std::string& name, const Length& maxObjectSize) : object_(name), maxObjectSize_(maxObjectSize), opened_(false) { if (!maxObjectSize_) { maxObjectSize_ = RadosCluster::instance().maxObjectSize(); } } RadosWriteHandle::~RadosWriteHandle() {} Length RadosWriteHandle::openForRead() { NOTIMP; } void RadosWriteHandle::openForWrite(const Length& length) { ASSERT(!opened_); written_ = 0; position_ = 0; part_ = 0; opened_ = true; } void RadosWriteHandle::openForAppend(const Length&) { NOTIMP; } long RadosWriteHandle::read(void* buffer, long length) { NOTIMP; } long RadosWriteHandle::write(const void* buffer, long length) { std::cout << "RadosWriteHandle::write " << length << std::endl; ASSERT(opened_); if (length == 0) { return 0; } long result = 0; const char* buf = reinterpret_cast(buffer); while (length > 0) { Length len = std::min(Length(maxObjectSize_ - Length(written_)), Length(length)); long l = (long)len; ASSERT(len == Length(l)); if (l == 0) { ASSERT(handle_); handle_->close(); handle_.reset(0); written_ = 0; continue; } std::cout << "RadosWriteHandle::write " << len << " - " << maxObjectSize_ << " - " << written_ << std::endl; if (!handle_.get()) { RadosObject object(object_, part_++); std::cout << "RadosWriteHandle::write open " << object << std::endl; handle_.reset(new RadosHandle(object)); handle_->openForWrite(0); // TODO: use proper size } handle_->write(buf + result, l); written_ += l; result += l; length -= l; position_ += l; } return result; } void RadosWriteHandle::flush() { // NOTIMP; } void RadosWriteHandle::close() { if (handle_.get()) { handle_->close(); handle_.reset(0); } if (opened_) { RadosAttributes attrs; attrs.set("length", position_); attrs.set("parts", part_); attrs.set("maxsize", maxObjectSize_); RadosCluster::instance().attributes(object_, attrs); opened_ = false; } } void RadosWriteHandle::rewind() { NOTIMP; } Offset RadosWriteHandle::position() { return position_; } std::string RadosWriteHandle::title() const { return PathName::shorten(object_.str()); } } // namespace eckit eckit-2.0.7/src/eckit/io/rados/RadosHandle.h0000664000175000017500000000345115161702250020747 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 2019 #ifndef eckit_io_rados_RadosHandle_h #define eckit_io_rados_RadosHandle_h #include #include "eckit/io/DataHandle.h" #include "eckit/io/rados/RadosCluster.h" #include "eckit/io/rados/RadosObject.h" namespace eckit { class RadosHandle : public eckit::DataHandle { public: // methods RadosHandle(const RadosObject&); RadosHandle(const std::string&); RadosHandle(Stream&); ~RadosHandle() override; // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } std::string title() const; public: // methods Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; Offset position() override; Length estimate() override; void print(std::ostream&) const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } private: // members RadosObject object_; uint64_t offset_; bool opened_; bool write_; void open(); static ClassSpec classSpec_; static Reanimator reanimator_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/rados/RadosReadHandle.cc0000664000175000017500000000534415161702250021704 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/rados/RadosReadHandle.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/MultiHandle.h" #include "eckit/io/rados/RadosAttributes.h" #include "eckit/io/rados/RadosCluster.h" #include "eckit/io/rados/RadosHandle.h" namespace eckit { ClassSpec RadosReadHandle::classSpec_ = { &DataHandle::classSpec(), "RadosReadHandle", }; Reanimator RadosReadHandle::reanimator_; void RadosReadHandle::print(std::ostream& s) const { s << "RadosReadHandle[" << object_ << ']'; } void RadosReadHandle::encode(Stream& s) const { DataHandle::encode(s); s << object_; } RadosReadHandle::RadosReadHandle(Stream& s) : DataHandle(s), object_(s) {} RadosReadHandle::RadosReadHandle(const std::string& name) : object_(name) {} RadosReadHandle::~RadosReadHandle() {} Length RadosReadHandle::estimate() { if (length_ == Length(0)) { RadosAttributes attr = RadosCluster::instance().attributes(object_); ASSERT(attr.get("length", length_)); } return length_; } Length RadosReadHandle::openForRead() { ASSERT(!handle_); RadosAttributes attr = RadosCluster::instance().attributes(object_); std::cout << "Attributes for " << object_ << " ===> " << attr << std::endl; ASSERT(attr.get("length", length_)); ASSERT(attr.get("parts", parts_)); handle_.reset(new MultiHandle()); for (size_t i = 0; i < parts_; ++i) { (*handle_) += new RadosHandle(RadosObject(object_, i)); } Length len = handle_->openForRead(); std::cout << "RadosReadHandle::openForRead attr=" << length_ << ", open=" << len << std::endl; ASSERT(len == length_); return length_; } void RadosReadHandle::openForWrite(const Length& length) { NOTIMP; } void RadosReadHandle::openForAppend(const Length&) { NOTIMP; } long RadosReadHandle::read(void* buffer, long length) { ASSERT(handle_); return handle_->read(buffer, length); } long RadosReadHandle::write(const void* buffer, long length) { NOTIMP; } void RadosReadHandle::flush() { NOTIMP; } void RadosReadHandle::close() { if (handle_) { handle_->close(); handle_.reset(0); } } void RadosReadHandle::rewind() { NOTIMP; } Offset RadosReadHandle::position() { NOTIMP; } std::string RadosReadHandle::title() const { return PathName::shorten(object_.str()); } } // namespace eckit eckit-2.0.7/src/eckit/io/rados/RadosReadHandle.h0000664000175000017500000000347215161702250021546 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 2019 #ifndef eckit_io_rados_RadosReadHandle_h #define eckit_io_rados_RadosReadHandle_h #include #include "eckit/io/DataHandle.h" #include "eckit/io/rados/RadosObject.h" namespace eckit { class MultiHandle; class RadosReadHandle : public eckit::DataHandle { public: // methods RadosReadHandle(const RadosObject&); RadosReadHandle(const std::string&); RadosReadHandle(Stream&); ~RadosReadHandle() override; // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } std::string title() const; public: // methods Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; Offset position() override; Length estimate() override; void print(std::ostream&) const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } private: // members RadosObject object_; Length length_; size_t parts_; std::unique_ptr handle_; static ClassSpec classSpec_; static Reanimator reanimator_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/rados/RadosWriteHandle.h0000664000175000017500000000362315161702250021763 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 2019 #ifndef eckit_io_rados_RadosWriteHandle_h #define eckit_io_rados_RadosWriteHandle_h #include #include "eckit/io/DataHandle.h" #include "eckit/io/rados/RadosObject.h" namespace eckit { class RadosWriteHandle : public eckit::DataHandle { public: // methods RadosWriteHandle(const RadosObject&, const Length& maxObjectSize = 0); RadosWriteHandle(const std::string&, const Length& maxObjectSize = 0); RadosWriteHandle(Stream&); ~RadosWriteHandle() override; // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } std::string title() const; public: // methods Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; Offset position() override; void print(std::ostream&) const override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } private: // members RadosObject object_; Length maxObjectSize_; size_t written_; Offset position_; size_t part_; bool opened_; std::unique_ptr handle_; static ClassSpec classSpec_; static Reanimator reanimator_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/rados/RadosCluster.h0000664000175000017500000000447315161702250021202 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 2019 #ifndef eckit_io_rados_RadosCluster_h #define eckit_io_rados_RadosCluster_h #include #include #include #include #include #include "eckit/io/Length.h" namespace eckit { class RadosObject; class RadosAttributes; class RadosIOCtx; #define RADOS_CALL(a) eckit::rados_call(a, #a, __FILE__, __LINE__, __func__) class RadosCluster { public: rados_ioctx_t& ioCtx(const std::string& pool) const; rados_ioctx_t& ioCtx(const RadosObject& object) const; Length maxObjectSize() const; rados_t cluster() const { return cluster_; } void ensurePool(const std::string& pool) const; void ensurePool(const RadosObject& object) const; void attributes(const RadosObject&, const RadosAttributes&) const; RadosAttributes attributes(const RadosObject&) const; bool exists(const RadosObject&) const; Length size(const RadosObject&) const; void remove(const RadosObject&) const; void truncate(const RadosObject&, const Length& = 0) const; time_t lastModified(const RadosObject&) const; // For multi-object items void removeAll(const RadosObject&) const; static const RadosCluster& instance(); private: RadosCluster(); ~RadosCluster(); private: rados_t cluster_; mutable std::map ctx_; void reset(); public: static void error(int code, const char* msg, const char* file, int line, const char* func); }; static inline int rados_call(int code, const char* msg, const char* file, int line, const char* func) { std::cout << "RADOS_CALL => " << msg << std::endl; if (code < 0) { std::cout << "RADOS_FAIL !! " << msg << std::endl; RadosCluster::error(code, msg, file, line, func); } std::cout << "RADOS_CALL <= " << msg << std::endl; return code; } } // namespace eckit #endif eckit-2.0.7/src/eckit/io/rados/RadosHandle.cc0000664000175000017500000000666215161702250021114 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/rados/RadosHandle.h" #include "eckit/exception/Exceptions.h" namespace eckit { ClassSpec RadosHandle::classSpec_ = { &DataHandle::classSpec(), "RadosHandle", }; Reanimator RadosHandle::reanimator_; void RadosHandle::print(std::ostream& s) const { s << "RadosHandle[" << object_ << ']'; } void RadosHandle::encode(Stream& s) const { DataHandle::encode(s); s << object_; } RadosHandle::RadosHandle(Stream& s) : DataHandle(s), object_(s), offset_(0), opened_(false), write_(false) {} RadosHandle::RadosHandle(const RadosObject& object) : object_(object), offset_(0), opened_(false), write_(false) {} RadosHandle::RadosHandle(const std::string& object) : object_(object), offset_(0), opened_(false), write_(false) {} RadosHandle::~RadosHandle() { // std::cout << "RadosHandle::~RadosHandle " << object_ << std::endl; if (opened_) { close(); } } void RadosHandle::open() { // std::cout << "RadosHandle::open " << object_ << std::endl; ASSERT(!opened_); offset_ = 0; opened_ = true; } Length RadosHandle::estimate() { return RadosCluster::instance().size(object_); } Length RadosHandle::openForRead() { // std::cout << "RadosHandle::openForRead " << object_ << std::endl; open(); write_ = false; return RadosCluster::instance().size(object_); } void RadosHandle::openForWrite(const Length& length) { // std::cout << "RadosHandle::openForWrite " << object_ << " " << length << std::endl; RadosCluster::instance().ensurePool(object_); RadosCluster::instance().truncate(object_); open(); write_ = true; } void RadosHandle::openForAppend(const Length&) { NOTIMP; } long RadosHandle::read(void* buffer, long length) { // std::cout << "RadosHandle::read " << object_ << " " << length << std::endl; ASSERT(opened_); ASSERT(!write_); long maxLength = RadosCluster::instance().maxObjectSize(); long readLength = length > maxLength ? maxLength : length; int len = RADOS_CALL(rados_read(RadosCluster::instance().ioCtx(object_), object_.oid().c_str(), reinterpret_cast(buffer), readLength, offset_)); // ASSERT(len > 0); offset_ += len; return len; } long RadosHandle::write(const void* buffer, long length) { ASSERT(length); // std::cout << "RadosHandle::write " << object_ << " " << length << std::endl; ASSERT(opened_); ASSERT(write_); RADOS_CALL(rados_write(RadosCluster::instance().ioCtx(object_), object_.oid().c_str(), reinterpret_cast(buffer), length, offset_)); offset_ += length; return length; } void RadosHandle::flush() { // NOTIMP; } void RadosHandle::close() { std::cout << "RadosHandle::close " << object_ << std::endl; ASSERT(opened_); opened_ = false; } void RadosHandle::rewind() { offset_ = 0; } Offset RadosHandle::position() { return offset_; } std::string RadosHandle::title() const { return PathName::shorten(object_.str()); } } // namespace eckit eckit-2.0.7/src/eckit/io/HandleBuf.h0000664000175000017500000000256315161702250017306 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File HandleBuf.h // Baudouin Raoult - ECMWF Mar 97 #ifndef eckit_HandleBuf_h #define eckit_HandleBuf_h #include "eckit/io/DataHandle.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class HandleBuf : public std::streambuf { public: // -- Contructors HandleBuf(DataHandle& handle, bool throwOnError = false); // -- Destructor ~HandleBuf(); private: // No copy allowed HandleBuf(const HandleBuf&); HandleBuf& operator=(const HandleBuf&); // -- Members char in_[1]; char out_[80]; DataHandle& handle_; bool throwOnError_; // -- Overridden methods // From streambuf virtual int overflow(int c); virtual int underflow(); virtual int sync(); // virtual int uflow(); }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/MemoryHandle.h0000664000175000017500000000435515161702250020043 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_filesystem_MemoryHandle_h #define eckit_filesystem_MemoryHandle_h #include "eckit/io/DataHandle.h" namespace eckit { class Buffer; //---------------------------------------------------------------------------------------------------------------------- class MemoryHandle : public DataHandle { public: MemoryHandle(const Buffer&); MemoryHandle(Buffer&); MemoryHandle(const void* address, size_t size); MemoryHandle(void* address, size_t size); MemoryHandle(size_t size = 1024 * 1024, bool grow = true); ~MemoryHandle() override; /// Access the underlying buffer /// Size is provided by both the non-const overriden virtual function size() /// and the const version (available only on this class) const void* data() const; std::string str() const; Length size() const; // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; void print(std::ostream&) const override; void skip(const Length&) override; Offset seek(const Offset&) override; bool canSeek() const override { return true; } Length size() override; Length estimate() override; Offset position() override; DataHandle* clone() const override; private: // members char* address_; size_t size_; size_t capacity_; bool opened_; bool readOnly_; bool read_; bool grow_; bool owned_; Offset position_; std::string title() const override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/Pipeline.h0000664000175000017500000000346615161702250017226 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Pipeline.h // Baudouin Raoult - ECMWF Feb 97 #ifndef eckit_Pipeline_h #define eckit_Pipeline_h #include "eckit/io/DataHandle.h" #include "eckit/io/Length.h" #include "eckit/io/TransferWatcher.h" #include "eckit/thread/Mutex.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class Pipeline { public: // -- Contructors Pipeline(TransferWatcher& = TransferWatcher::dummy()); Pipeline(const Pipeline&) = delete; Pipeline& operator=(const Pipeline&) = delete; Pipeline(Pipeline&&) = delete; Pipeline& operator=(Pipeline&&) = delete; // -- Destructor virtual ~Pipeline(); // -- Methods Length copy(DataHandle&, DataHandle&); bool error(); void error(const std::string&); void restart(RestartTransfer&); private: virtual void execute(DataHandle& in, DataHandle& out) = 0; private: // members Mutex mutex_; long count_; long bufSize_; Length inBytes_; Length outBytes_; bool error_; std::string why_; bool restart_; Offset restartFrom_; TransferWatcher& watcher_; // -- Friends friend class PipelineExecutor; friend class PipelineReader; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/EmptyHandle.h0000664000175000017500000000406015161702250017662 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/EmptyHandle.h // Manuel Fuentes - ECMWF Jul 96 #ifndef eckit_filesystem_EmptyHandle_h #define eckit_filesystem_EmptyHandle_h #include "eckit/io/DataHandle.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class EmptyHandle : public DataHandle { public: // -- Contructors EmptyHandle() {} EmptyHandle(Stream& s) : DataHandle(s) {} // -- Destructor ~EmptyHandle() {} // -- Overridden methods // From DataHandle Length openForRead() override { return 0; } void openForWrite(const Length&) override {} void openForAppend(const Length&) override {} long read(void*, long) override { return 0; } long write(const void*, long n) override { return n; } void close() override {} void flush() override {} Offset seek(const Offset&) override { return 0; } Offset position() override { return 0; } void rewind() override {} void print(std::ostream& s) const override { s << "Empty Handle"; } bool isEmpty() const override { return true; } bool canSeek() const override { return true; } // From Streamable void encode(Stream& s) const override { DataHandle::encode(s); } const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- Class members static ClassSpec classSpec_; static Reanimator reanimator_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/HandleBuf.cc0000664000175000017500000000416415161702250017443 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/HandleBuf.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- HandleBuf::HandleBuf(DataHandle& handle, bool throwOnError) : handle_(handle), throwOnError_(throwOnError) { setg(in_, in_, in_); setp(out_, out_ + sizeof(out_)); } HandleBuf::~HandleBuf() { sync(); } int HandleBuf::sync() { int len = pptr() - pbase(); if (len != 0) { int written = handle_.write(pbase(), len); if (len != written) { if (throwOnError_) { std::ostringstream oss; oss << "HandleBuf: failed to write to " << handle_; throw WriteError(oss.str()); } return EOF; } } setp(pbase(), epptr()); return 0; } int HandleBuf::overflow(int c) { if (sync()) { return EOF; } if (c == EOF) { return 0; } sputc(c); return 0; } int HandleBuf::underflow() { if (gptr() < egptr()) { return *(unsigned char*)gptr(); } int n = handle_.read(in_, sizeof(in_)); if (n == EOF || n == 0) { if (throwOnError_) { std::ostringstream oss; oss << "HandleBuf: failed to read from " << handle_; throw ReadError(oss.str()); } return EOF; } setg(in_, in_, in_ + n); return *(unsigned char*)gptr(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/FTPHandle.h0000664000175000017500000000377115161702250017225 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/FTPHandle.h // Manuel Fuentes - ECMWF May 96 #ifndef eckit_filesystem_FTPHandle_h #define eckit_filesystem_FTPHandle_h #include "eckit/io/DataHandle.h" #include "eckit/net/TCPClient.h" #include "eckit/net/TCPSocket.h" namespace eckit { class FTPHandle : public DataHandle { public: // -- Exceptions class FTPError : public std::exception { virtual const char* what() const noexcept; }; // -- Contructors FTPHandle(const std::string&, const std::string&, int port = 21); FTPHandle(Stream&); // -- Destructor ~FTPHandle() {} // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void rewind() override; void print(std::ostream&) const override; bool canSeek() const override { return false; } // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- Members std::string remote_; std::string host_; int port_; net::TCPClient cmds_; net::TCPSocket data_; // -- Methods void ftpCommand(const std::string&); std::string readLine(); void open(const std::string&); // -- Class members static ClassSpec classSpec_; static Reanimator reanimator_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/RawFileHandle.h0000664000175000017500000000315715161702250020123 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Dec 2013 #ifndef eckit_filesystem_RawFileHandle_h #define eckit_filesystem_RawFileHandle_h #include "eckit/io/DataHandle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class RawFileHandle : public DataHandle { public: RawFileHandle(const std::string& path, bool overwrite = false); ~RawFileHandle(); Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void print(std::ostream&) const override; Length size() override; Length estimate() override; Offset position() override; Offset seek(const Offset&) override; bool canSeek() const override { return true; } void skip(const Length&) override; void encode(Stream&) const override; private: std::string path_; bool overwrite_; int fd_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/SharedHandle.h0000664000175000017500000000504615161702250017777 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Partio/FileHandle.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_filesystem_SharedHandle_h #define eckit_filesystem_SharedHandle_h #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/types/Types.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class SharedHandle : public DataHandle { public: // -- Contructors SharedHandle(DataHandle& handle); // -- Destructor ~SharedHandle(); // -- Methods // -- Overridden methods // From DataHandle void print(std::ostream& s) const override; Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; Length size() override; Length estimate() override; Offset position() override; Offset seek(const Offset&) override; bool canSeek() const override; void skip(const Length&) override; void rewind() override; void restartReadFrom(const Offset&) override; void restartWriteFrom(const Offset&) override; DataHandle* clone() const override; using DataHandle::saveInto; Length saveInto(DataHandle& other, TransferWatcher& watcher) override; std::string name() const override; bool compress(bool) override; bool merge(DataHandle*) override; bool isEmpty() const override; bool moveable() const override; void toLocal(Stream& s) const override; DataHandle* toLocal() override; void toRemote(Stream& s) const override; void selectMover(MoverTransferSelection&, bool read) const override; std::string title() const override; void collectMetrics(const std::string& what) const override; // Tag for metrics collection private: // -- Members DataHandle& handle_; // -- Methods }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/SeekableHandle.cc0000664000175000017500000000504215161702250020436 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/SeekableHandle.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- SeekableHandle::SeekableHandle(PeekHandle* h) : owned_(true), handle_(h), seekableStart_(0), position_(0) {} SeekableHandle::SeekableHandle(PeekHandle& h) : owned_(false), handle_(&h), seekableStart_(0), position_(0) {} SeekableHandle::~SeekableHandle() { if (owned_) { delete handle_; } } Length SeekableHandle::openForRead() { position_ = 0; seekableStart_ = 0; return handle_->openForRead(); } void SeekableHandle::close() { handle_->close(); } void SeekableHandle::print(std::ostream& s) const { s << "SeekableHandle["; handle_->print(s); s << ']'; } Length SeekableHandle::estimate() { return handle_->estimate(); } void SeekableHandle::skip(const Length& len) { ASSERT(position_ + len <= seekableStart_ + Length(handle_->peeked())); position_ += len; } long SeekableHandle::read(void* buffer, long length) { long len = handle_->peek(buffer, length, position_ - seekableStart_); ASSERT(len >= 0); position_ += len; return len; } void SeekableHandle::rewind() { ASSERT(seekableStart_ == Offset(0)); position_ = 0; } Offset SeekableHandle::seek(const Offset& off) { ASSERT(off >= seekableStart_); ASSERT(off <= seekableStart_ + Length(handle_->peeked())); position_ = off; return position_; } bool SeekableHandle::canSeek() const { return true; } Offset SeekableHandle::position() { return position_; } std::string SeekableHandle::title() const { return std::string("{") + handle_->title() + "}"; } void SeekableHandle::collectMetrics(const std::string& what) const { handle_->collectMetrics(what); } void SeekableHandle::clear() { Length peeked = handle_->peeked(); handle_->skip(peeked); seekableStart_ += peeked; position_ = seekableStart_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/FilePool.cc0000664000175000017500000000647715161702250017335 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ /// @author Tiago Quintino /// @date Dec 2015 #include "eckit/io/FilePool.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/DataHandle.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/MutexCond.h" #include "eckit/types/Types.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static void closeDataHandle(PathName&, DataHandle*& handle) { if (handle) { handle->close(); delete handle; handle = nullptr; } } static bool inUse(const std::map& store, const PathName& path) { return store.find(path) != store.end(); } //---------------------------------------------------------------------------------------------------------------------- FilePool::FilePool(size_t size) : cache_(size, &closeDataHandle) {} FilePool::~FilePool() {} DataHandle* FilePool::checkout(const PathName& path) { AutoLock lock(cond_); while (inUse(inUse_, path)) { cond_.wait(); } DataHandle* dh; const bool inCache = cache_.exists(path); if (inCache && path.exists()) { dh = cache_.extract(path); } else { // FileHandle is cached, but file has been removed on disk if (inCache) { cache_.remove(path); } dh = path.fileHandle(false); // append mode (no overwrite) dh->openForAppend(0); } ASSERT(dh); inUse_[path] = dh; return dh; } void FilePool::checkin(DataHandle* handle) { AutoLock lock(cond_); using iterator_type = std::map::iterator; for (iterator_type itr = inUse_.begin(); itr != inUse_.end(); ++itr) { if (itr->second == handle) { cache_.insert(itr->first, itr->second); inUse_.erase(itr); cond_.signal(); return; } } throw eckit::SeriousBug("Should have found a DataHandle in pool use", Here()); } bool FilePool::remove(const PathName& path) { AutoLock lock(cond_); while (inUse(inUse_, path)) { cond_.wait(); } return cache_.remove(path); } void FilePool::print(std::ostream& os) const { AutoLock lock(const_cast(*this).cond_); os << "FilePool(" << "inUse=" << inUse_ << ", " << "cache=" << cache_ << ")"; } size_t FilePool::size() const { AutoLock lock(cond_); return cache_.size(); } void FilePool::capacity(size_t size) { AutoLock lock(cond_); cache_.capacity(size); } size_t FilePool::capacity() const { AutoLock lock(cond_); return cache_.capacity(); } size_t FilePool::usage() const { AutoLock lock(cond_); return inUse_.size(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/CircularBuffer.h0000664000175000017500000000273215161702250020352 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date June 2017 #ifndef eckit_CircularBuffer_h #define eckit_CircularBuffer_h #include #include "eckit/eckit.h" #include "eckit/thread/Mutex.h" namespace eckit { // A simple class to implement buffers class CircularBuffer { public: // methods CircularBuffer(size_t size = 64 * 1024, size_t capactity = std::numeric_limits::max()); CircularBuffer(const CircularBuffer&) = delete; CircularBuffer& operator=(const CircularBuffer&) = delete; CircularBuffer(CircularBuffer&&) = delete; CircularBuffer& operator=(CircularBuffer&&) = delete; ~CircularBuffer(); size_t write(const void* buffer, size_t length); size_t read(void* buffer, size_t length); size_t length() const; size_t capacity() const; size_t size() const; void clear(); private: // members mutable Mutex mutex_; char* buffer_; size_t increment_; size_t size_; size_t capacity_; size_t pos_; size_t used_; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/cluster/0000775000175000017500000000000015161702250016760 5ustar alastairalastaireckit-2.0.7/src/eckit/io/cluster/ClusterNode.h0000664000175000017500000000255115161702250021363 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_ClusterNode_h #define eckit_ClusterNode_h #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Stream; class NodeInfo; class ClusterNode { public: ClusterNode(); ClusterNode(const ClusterNode&) = delete; ClusterNode& operator=(const ClusterNode&) = delete; ClusterNode(ClusterNode&&) = delete; ClusterNode& operator=(ClusterNode&&) = delete; virtual ~ClusterNode(); void heartbeat(); virtual const std::set& attributes() const; virtual int port() const = 0; virtual void initialise(Stream&) = 0; virtual void refresh(Stream&) = 0; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/cluster/ClusterDisks.cc0000664000175000017500000003405415161702250021714 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @date Jun 2011 /// @author Baudouin Raoult /// @author Tiago Quintino #include #include #include #include "eckit/config/Resource.h" #include "eckit/container/MappedArray.h" #include "eckit/container/SharedMemArray.h" #include "eckit/filesystem/FileSpace.h" #include "eckit/filesystem/LocalPathName.h" #include "eckit/io/cluster/ClusterDisks.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/log/JSON.h" #include "eckit/memory/Zero.h" #include "eckit/system/SystemInfo.h" #include "eckit/thread/AutoLock.h" #include "eckit/utils/Tokenizer.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class ClusterDisk { bool active_; bool offLine_; time_t lastSeen_; char node_[256]; char type_[256]; char path_[2048]; public: ClusterDisk(const std::string& node, const std::string& type, const std::string& path) : active_(true), offLine_(false), lastSeen_(::time(nullptr)) { zero(node_); strncpy(node_, node.c_str(), sizeof(node_) - 1); zero(type_); strncpy(type_, type.c_str(), sizeof(type_) - 1); zero(path_); strncpy(path_, path.c_str(), sizeof(path_) - 1); } bool operator<(const ClusterDisk& other) const { if (compare(other) < 0) { return true; } return false; } int compare(const ClusterDisk& other) const { int cmp = ::strcmp(path_, other.path_); // n.b. We do NOT compare the node. It is legitimate for a path to move to // another node. // if (cmp == 0) cmp = ::strcmp(node_, other.node_); return cmp; } void active(bool on) { active_ = on; } void offLine(bool on) { offLine_ = on; } bool active() const { return active_; } bool offLine() const { return offLine_; } time_t lastSeen() const { return lastSeen_; } void lastSeen(time_t n) { lastSeen_ = n; } const char* node() const { return node_; } const char* type() const { return type_; } const char* path() const { return path_; } friend std::ostream& operator<<(std::ostream&, const ClusterDisk&); void json(JSON& s) const { s.startObject(); s << "lastSeen" << lastSeen_; s << "offLine" << offLine_; s << "node" << node_; s << "type" << type_; s << "path" << path_; s.endObject(); } void send(Stream& s) const { unsigned long long t = lastSeen_; s << t; s << offLine_; s << node_; s << type_; s << path_; } void receive(Stream& s) { unsigned long long t; std::string x; s >> t; lastSeen_ = t; s >> offLine_; s >> x; zero(node_); ::strncpy(node_, x.c_str(), sizeof(node_) - 1); s >> x; zero(type_); ::strncpy(type_, x.c_str(), sizeof(type_) - 1); s >> x; zero(path_); ::strncpy(path_, x.c_str(), sizeof(path_) - 1); active_ = true; } }; std::ostream& operator<<(std::ostream& s, const ClusterDisk& d) { s << "ClusterDisk[" << d.node_ << "," << d.type_ << "," << d.path_ << "," << (::time(nullptr) - d.lastSeen_) << "," << (d.offLine_ ? "off" : "on") << "-line" << "]"; return s; } inline unsigned long version(ClusterDisk*) { return 1; } //---------------------------------------------------------------------------------------------------------------------- class DiskArray { public: using iterator = ClusterDisk*; using const_iterator = const ClusterDisk*; DiskArray() = default; DiskArray(const DiskArray&) = delete; DiskArray& operator=(const DiskArray&) = delete; DiskArray(DiskArray&&) = delete; DiskArray& operator=(DiskArray&&) = delete; virtual ~DiskArray() {} virtual void sync() = 0; virtual void lock() = 0; virtual void unlock() = 0; virtual iterator begin() = 0; virtual iterator end() = 0; virtual const_iterator begin() const = 0; virtual const_iterator end() const = 0; virtual unsigned long size() = 0; virtual ClusterDisk& operator[](unsigned long n) = 0; }; class MemoryMappedDiskArray : public DiskArray { virtual void sync() { map_.sync(); } virtual void lock() { map_.lock(); } virtual void unlock() { map_.unlock(); } virtual iterator begin() { return map_.begin(); } virtual iterator end() { return map_.end(); } virtual const_iterator begin() const { return map_.begin(); } virtual const_iterator end() const { return map_.end(); } virtual unsigned long size() { return map_.size(); } virtual ClusterDisk& operator[](unsigned long n) { return map_[n]; } MappedArray map_; public: MemoryMappedDiskArray(const PathName& path, unsigned long size) : DiskArray(), map_(path, size) {} }; class SharedMemoryDiskArray : public DiskArray { virtual void sync() { map_.sync(); } virtual void lock() { map_.lock(); } virtual void unlock() { map_.unlock(); } virtual iterator begin() { return map_.begin(); } virtual iterator end() { return map_.end(); } virtual const_iterator begin() const { return map_.begin(); } virtual const_iterator end() const { return map_.end(); } virtual unsigned long size() { return map_.size(); } virtual ClusterDisk& operator[](unsigned long n) { return map_[n]; } SharedMemArray map_; public: SharedMemoryDiskArray(const PathName& path, const std::string& name, unsigned long size) : DiskArray(), map_(path, name, size) {} }; static DiskArray* clusterDisks = nullptr; static pthread_once_t once = PTHREAD_ONCE_INIT; static void diskarray_init() { LocalPathName path("~/etc/cluster/disks"); // avoid recursion... size_t disksArraySize = Resource("disksArraySize", 10240); std::string diskArrayType = Resource("disksArrayType", "MemoryMapped"); if (diskArrayType == "MemoryMapped") { clusterDisks = new MemoryMappedDiskArray(path, disksArraySize); return; } if (diskArrayType == "SharedMemory") { std::string shmpath = eckit::system::SystemInfo::instance().userName() + "-etc-cluster-disks"; clusterDisks = new SharedMemoryDiskArray(path, shmpath, disksArraySize); return; } std::ostringstream oss; oss << "Invalid diskArrayType : " << diskArrayType << ", valid types are 'MemoryMapped' and 'SharedMemory'" << std::endl; throw eckit::BadParameter(oss.str(), Here()); } void ClusterDisks::reset() { pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); for (DiskArray::iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) { (*k).active(false); } } void ClusterDisks::cleanup() { /* pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); for(DiskArray::iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) if((*k).active() && (*k).offline()) { Log::info() << "Forget " << (*k) << std::endl; (*k).active(false); } */ reset(); } void ClusterDisks::forget(const NodeInfo& info) { if (info.name() == "marsfs") { time_t now = ::time(nullptr); pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); for (DiskArray::iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) { if (info.node() == (*k).node()) { (*k).active(false); } (*k).lastSeen(now); } } } void ClusterDisks::offLine(const NodeInfo& info) { if (info.name() == "marsfs") { time_t now = ::time(nullptr); pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); // cout << "=========== ClusterDisks::forget " << info << std::endl; for (DiskArray::iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) { if (info.node() == (*k).node()) { (*k).offLine(true); } (*k).lastSeen(now); } } } void ClusterDisks::update(const std::string& node, const std::string& type, const std::vector& disks) { pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); for (DiskArray::iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) { if (type == (*k).type() && node == (*k).node()) { (*k).active(false); } } for (std::vector::const_iterator j = disks.begin(); j != disks.end(); ++j) { ClusterDisk c(node, type, *j); DiskArray::iterator k = std::lower_bound(clusterDisks->begin(), clusterDisks->end(), c); if (k != clusterDisks->end() && c.compare(*k) == 0) { // Log::info() << "=========== Update " << node << "-" << type << " " << *j << std::endl; // TODO check for change of node or type *k = c; } else { ASSERT(!(*clusterDisks)[0].active()); // Log::info() << "================ New " << node << "-" << type << " " << * j << std::endl; (*clusterDisks)[0] = c; std::sort(clusterDisks->begin(), clusterDisks->end()); } } } void ClusterDisks::list(std::ostream& out) { pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); for (DiskArray::const_iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) { if ((*k).active()) { out << *k << std::endl; } } } void ClusterDisks::json(JSON& j) { pthread_once(&once, diskarray_init); j.startList(); AutoLock lock(*clusterDisks); for (DiskArray::const_iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) { if ((*k).active()) { (*k).json(j); } } j.endList(); } time_t ClusterDisks::lastModified(const std::string& type) { pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); time_t last = 0; for (DiskArray::const_iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) { if ((*k).active() && ((*k).type() == type)) { last = std::max(last, (*k).lastSeen()); } } return last; } void ClusterDisks::load(const std::string& type, std::vector& disks) { pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); for (DiskArray::const_iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) { if ((*k).active() && ((*k).type() == type)) { disks.push_back(std::string("marsfs://") + (*k).node() + (*k).path()); } } } std::string ClusterDisks::node(const std::string& path) { pthread_once(&once, diskarray_init); DiskArray::const_iterator j = clusterDisks->end(); AutoLock lock(*clusterDisks); for (DiskArray::const_iterator k = clusterDisks->begin(); k != clusterDisks->end(); ++k) { if ((*k).active() && (path.find((*k).path()) == 0)) { if (j != clusterDisks->end()) { std::ostringstream os; os << "Two nodes found for [" << path << "] " << "marsfs://" << (*j).node() << "/" << (*j).path() << "and " << "marsfs://" << (*k).node() << "/" << (*k).path(); throw SeriousBug(os.str()); } j = k; } } if (j == clusterDisks->end()) { // Look for local names // This is ineficent, but is should be called very rarely if (LocalPathName(path).exists()) { return NodeInfo::thisNode().node(); } LocalPathName df("~/etc/disks/df"); std::ifstream in(df.localPath()); char line[1024]; while (in.getline(line, sizeof(line))) { if (line[0] != 0 && line[0] != '#') { Tokenizer tokenize(", \t"); std::vector tokens; tokenize(line, tokens); if (tokens.size() == 2) { const FileSpace& fs = FileSpace::lookUp(tokens[0]); const std::vector& v = fs.fileSystems(); for (size_t j = 0; j < v.size(); ++j) { if (path.find(v[j].asString()) == 0) { Log::info() << "ClusterDisks::node [" << path << "] is on " << v[j] << std::endl; return "local"; } } } } } std::ostringstream os; os << "No node found for [" << path << "]"; throw SeriousBug(os.str()); } return (*j).node(); } void ClusterDisks::send(Stream& s) { pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); for (const ClusterDisk& disk : *clusterDisks) { if (disk.active()) { s << bool(true); disk.send(s); } } s << bool(false); } void ClusterDisks::receive(Stream& s) { pthread_once(&once, diskarray_init); AutoLock lock(*clusterDisks); for (ClusterDisk& disk : *clusterDisks) { disk.active(false); } bool more; DiskArray::iterator k = clusterDisks->begin(); while (true) { s >> more; if (!more) { break; } ASSERT(k != clusterDisks->end()); (*k).receive(s); ++k; } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/cluster/NodeInfo.h0000664000175000017500000000516515161702250020641 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_io_NodeInfo_h #define eckit_io_NodeInfo_h #include "eckit/runtime/TaskID.h" #include "eckit/types/Types.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class NodeInfo { public: // methods NodeInfo(); void port(int p) { port_ = p; } int port() const { return port_; } void active(bool a) { active_ = a; } bool active() const { return active_; } void host(const std::string& h) { host_ = h; } const std::string& host() const { return host_; } void name(const std::string& h) { name_ = h; } const std::string& name() const { return name_; } void user(const std::string& h) { user_ = h; } const std::string& user() const { return user_; } void node(const std::string& h) { node_ = h; } const std::string& node() const { return node_; } void attribute(const std::string& a) { attributes_.insert(a); } void attributes(const std::set& a) { attributes_ = a; } const std::set& attributes() const { return attributes_; } bool supportsAttributes(const std::set& attrs) const; void id(TaskID p) { id_ = p; } TaskID id() const { return id_; } void task(long p) { task_ = p; } long task() const { return task_; } NodeInfo& init(); static NodeInfo& thisNode(); static NodeInfo acceptLogin(Stream&); static NodeInfo sendLogin(Stream&); private: // members std::string name_; std::string node_; std::string user_; std::string host_; std::set attributes_; int port_; bool active_; TaskID id_; long task_; private: // methods void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const NodeInfo& p) { p.print(s); return s; } friend void operator<<(Stream&, const NodeInfo&); friend void operator>>(Stream&, NodeInfo&); }; // Used by MappedArray inline unsigned long version(NodeInfo*) { return 1; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/cluster/ClusterDisks.h0000664000175000017500000000303215161702250021546 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @date Jun 2011 /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef eckit_ClusterDisks_h #define eckit_ClusterDisks_h #include #include #include #include "eckit/io/cluster/NodeInfo.h" #include "eckit/log/JSON.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class NodeInfo; class JSON; class ClusterDisks { public: static void reset(); static void cleanup(); static void offLine(const NodeInfo&); static void forget(const NodeInfo&); static void update(const std::string&, const std::string&, const std::vector&); static void list(std::ostream& out); static void json(JSON& out); static void send(Stream& s); static void receive(Stream& s); static time_t lastModified(const std::string&); static void load(const std::string&, std::vector&); static std::string node(const std::string& path); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/cluster/NodeInfo.cc0000664000175000017500000001027215161702250020772 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/config/Resource.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/runtime/Main.h" #include "eckit/runtime/Monitor.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" #include "eckit/thread/ThreadSingleton.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static StaticMutex local_mutex; NodeInfo::NodeInfo() : port_(0), active_(false), id_(0), task_(-1) {} NodeInfo& NodeInfo::init() { AutoLock lock(local_mutex); if (!name_.length()) { static std::string myNode = Resource("node", ""); static std::string myHost = Resource("host", ""); static std::string myUser = Resource("user", ""); name_ = Main::instance().name(); host_ = myHost; if (host_.length() == 0) { myHost = host_ = eckit::Main::hostname(); host_ = host_.substr(0, host_.find(".")); } node_ = myNode; if (node_.length() == 0) { node_ = host_; myNode = node_; } user_ = myUser; if (user_.length() == 0) { user_ = ""; char buf[4096]; struct passwd pwbuf; struct passwd* pwbufp = nullptr; SYSCALL(getpwuid_r(getuid(), &pwbuf, buf, sizeof(buf), &pwbufp)); if (pwbufp) { user_ = pwbuf.pw_name; } myUser = user_; } task_ = Monitor::instance().self(); } return *this; } static ThreadSingleton n; NodeInfo& NodeInfo::thisNode() { return n.instance().init(); } void operator<<(Stream& s, const NodeInfo& info) { s << info.user(); s << info.name(); s << info.node(); s << info.host(); s << info.port(); s << info.id(); s << info.task(); s << info.attributes(); } void operator>>(Stream& s, NodeInfo& info) { std::string p; int l; s >> p; info.user(p); s >> p; info.name(p); s >> p; info.node(p); s >> p; info.host(p); s >> l; info.port(l); TaskID id; s >> id; info.id(id); long task; s >> task; info.task(task); std::set a; s >> a; info.attributes(a); } void NodeInfo::print(std::ostream& s) const { s << "[" << name_ << ":" << std::setfill('0') << std::setw(3) << task_ << std::setfill(' ') << "," << node_ << "@" << host_ << ":" << port_ << "," << user_ << "]"; } NodeInfo NodeInfo::acceptLogin(Stream& s) { NodeInfo remote; NodeInfo& here = thisNode(); s >> remote; if (here.user() == remote.user()) { s << here; } else { std::ostringstream os; os << "User mismatch: " << here << " " << remote; s << Exception(os.str()); } Log::info() << "Connection established " << here << " <=> " << remote << std::endl; return remote; } NodeInfo NodeInfo::sendLogin(Stream& s) { NodeInfo remote; NodeInfo& here = thisNode(); s << here; s >> remote; if (here.user() != remote.user()) { std::ostringstream os; os << "User mismatch: " << here << " " << remote; s << Exception(os.str()); } Log::info() << "Connection established " << here << " <=> " << remote << std::endl; return remote; } bool NodeInfo::supportsAttributes(const std::set& attrs) const { for (const auto& a : attrs) { if (attributes_.find(a) == attributes_.end()) { return false; } } return true; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/cluster/ClusterNodes.cc0000664000175000017500000003335015161702250021705 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File ClusterNodes.cc // Baudouin Raoult - (c) ECMWF Jul 11 #include #include "eckit/config/EtcTable.h" #include "eckit/config/Resource.h" #include "eckit/container/MappedArray.h" #include "eckit/io/cluster/ClusterNodes.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/log/JSON.h" #include "eckit/memory/Zero.h" #include "eckit/net/IPAddress.h" #include "eckit/thread/AutoLock.h" #include "eckit/utils/Clock.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static const constexpr int MAX_NODE_ATTRIBUTES = 16; class ClusterNodeEntry { bool active_; time_t lastSeen_; bool offLine_; char node_[256]; char type_[256]; char host_[256]; int nattrs_; char attributes_[MAX_NODE_ATTRIBUTES][256]; int port_; ClusterNodeEntry(const std::string& node, const std::string& type, const std::string& host, int port, const std::set& attributes) : active_(true), lastSeen_(Clock::now()), offLine_(false), port_(port) { zero(node_); strncpy(node_, node.c_str(), sizeof(node_) - 1); zero(type_); strncpy(type_, type.c_str(), sizeof(type_) - 1); zero(host_); strncpy(host_, host.c_str(), sizeof(host_) - 1); ASSERT(attributes.size() <= MAX_NODE_ATTRIBUTES); zero(attributes_); nattrs_ = 0; for (const auto& a : attributes) { strncpy(attributes_[nattrs_++], a.c_str(), sizeof(attributes_[0]) - 1); } } public: ClusterNodeEntry(const NodeInfo& info) : ClusterNodeEntry(info.node(), info.name(), info.host(), info.port(), info.attributes()) {} NodeInfo asNodeInfo() const { NodeInfo info; info.name(type()); info.node(node()); info.host(host()); info.port(port()); info.active(!offLine()); info.attributes(attributes()); return info; } bool operator<(const ClusterNodeEntry& other) const { if (strcmp(node_, other.node_) < 0) { return true; } if (strcmp(type_, other.type_) < 0) { return true; } return false; } void send(Stream& s) const { unsigned long long t = lastSeen_; s << t; s << offLine_; s << node_; s << type_; s << host_; s << nattrs_; for (int i = 0; i < nattrs_; ++i) { s << attributes_[i]; } s << port_; } void json(JSON& s) const { s.startObject(); s << "lastSeen" << lastSeen_; s << "offLine" << offLine_; s << "available" << available(); s << "node" << node_; s << "type" << type_; s << "host" << host_; s << "attributes"; s.startList(); for (int i = 0; i < nattrs_; ++i) { s << attributes_[i]; } s.endList(); s << "port" << port_; s.endObject(); } void receive(Stream& s) { unsigned long long t; std::string x; s >> t; lastSeen_ = t; s >> offLine_; s >> x; zero(node_); strncpy(node_, x.c_str(), sizeof(node_) - 1); s >> x; zero(type_); strncpy(type_, x.c_str(), sizeof(type_) - 1); s >> x; zero(host_); strncpy(host_, x.c_str(), sizeof(host_) - 1); s >> nattrs_; ASSERT(nattrs_ >= 0 && nattrs_ <= MAX_NODE_ATTRIBUTES); zero(attributes_); for (int i = 0; i < nattrs_; ++i) { s >> x; strncpy(attributes_[i], x.c_str(), sizeof(attributes_[0]) - 1); } s >> port_; active_ = true; } bool available() const { static long maxNodeLastSeen = Resource("maxNodeLastSeen", 60); return ((Clock::now() - lastSeen_) <= maxNodeLastSeen) && !offLine_; } void active(bool on) { active_ = on; } void offLine(bool on) { offLine_ = on; } bool active() const { return active_; } bool offLine() const { return offLine_; } time_t lastSeen() const { return lastSeen_; } void lastSeen(time_t n) { lastSeen_ = n; } const char* node() const { return node_; } const char* type() const { return type_; } const char* host() const { return host_; } void attributes(const std::set& attrs) { ASSERT(attrs.size() <= MAX_NODE_ATTRIBUTES); zero(attributes_); nattrs_ = 0; for (const auto& a : attrs) { strncpy(attributes_[nattrs_++], a.c_str(), sizeof(attributes_[0]) - 1); } } std::set attributes() const { std::set ret; for (int i = 0; i < nattrs_; ++i) { ret.insert(attributes_[i]); } return ret; } int attributesCount() const { return nattrs_; } bool hasAttributes(const std::set& attributes) const { for (const auto& a : attributes) { bool found = false; for (int i = 0; i < nattrs_; ++i) { if (a == attributes_[i]) { found = true; break; } } if (!found) { return false; } } return true; } void host(const std::string& h) { zero(host_); strncpy(host_, h.c_str(), sizeof(host_) - 1); } void port(int p) { port_ = p; } int port() const { return port_; } friend std::ostream& operator<<(std::ostream&, const ClusterNodeEntry&); }; std::ostream& operator<<(std::ostream& s, const ClusterNodeEntry& d) { s << "ClusterNodeEntry[" << d.node_ << "," << d.type_ << "," << d.host_ << ":" << d.port_ << "," << (Clock::now() - d.lastSeen_) << "," << (d.available() ? "available" : "not-available") << "," << (d.offLine_ ? "off" : "on") << "-line,{"; bool first = true; for (int i = 0; i < d.nattrs_; ++i) { s << (first ? "" : ",") << d.attributes_[i]; first = false; } s << "}]"; return s; } inline unsigned long version(ClusterNodeEntry*) { return 1; } using NodeArray = MappedArray; static NodeArray* nodeArray = nullptr; static pthread_once_t once = PTHREAD_ONCE_INIT; static std::set offsiteNodes_; static void init() { nodeArray = new NodeArray("~/etc/cluster/nodes", 1024); EtcKeyTable config("cluster/offsite", 1); if (config.exists()) { for (const auto& line : config.lines()) { offsiteNodes_.insert(line[0]); Log::info() << "Offsite nodes [" << line[0] << "]" << std::endl; } } } void ClusterNodes::reset() { pthread_once(&once, init); AutoLock lock(*nodeArray); for (NodeArray::iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { (*k).offLine(true); } } void ClusterNodes::cleanup() { pthread_once(&once, init); AutoLock lock(*nodeArray); for (NodeArray::iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active() && !(*k).available()) { Log::info() << "Forget " << (*k) << std::endl; (*k).active(false); } } } void ClusterNodes::forget(const NodeInfo& info) { pthread_once(&once, init); AutoLock lock(*nodeArray); for (NodeArray::iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if (info.node() == (*k).node()) { (*k).active(false); } } } void ClusterNodes::refresh(const NodeInfo& info) { pthread_once(&once, init); AutoLock lock(*nodeArray); time_t now = Clock::now(); for (NodeArray::iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active()) { if (info.node() == (*k).node() && info.name() == (*k).type()) { (*k).lastSeen(now); (*k).host(info.host()); (*k).port(info.port()); (*k).offLine(false); (*k).attributes(info.attributes()); return; } } } std::sort(nodeArray->begin(), nodeArray->end()); ASSERT(!(*nodeArray)[0].active()); (*nodeArray)[0] = ClusterNodeEntry(info); std::sort(nodeArray->begin(), nodeArray->end()); } NodeInfo ClusterNodes::lookUp(const std::string& type, const std::string& node) { pthread_once(&once, init); AutoLock lock(*nodeArray); for (const auto& k : *nodeArray) { if (k.active() && type == k.type() && node == k.node()) { return k.asNodeInfo(); } } if (offsite(type, node)) { throw UnexpectedState(type + "@" + node + " is offsite."); } throw SeriousBug(std::string("Cannot find info for ") + type + "@" + node); } NodeInfo ClusterNodes::any(const std::string& type, const std::set& attributes) { pthread_once(&once, init); AutoLock lock(*nodeArray); std::vector permitted; for (const ClusterNodeEntry& k : *nodeArray) { if (k.active() && k.available() && type == k.type()) { if (k.hasAttributes(attributes)) { permitted.push_back(&k); } } } if (permitted.empty()) { throw Retry(std::string("Cannot find any node for ") + type); } int choice = random() % permitted.size(); return permitted[choice]->asNodeInfo(); } bool ClusterNodes::available(const std::string& type, const std::string& node) { pthread_once(&once, init); AutoLock lock(*nodeArray); for (NodeArray::const_iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active() && type == (*k).type() && node == (*k).node()) { return (*k).available(); } } return false; } bool ClusterNodes::offsite(const std::string& type, const std::string& node) { pthread_once(&once, init); return offsiteNodes_.find(node) != offsiteNodes_.end(); } void ClusterNodes::offLine(const NodeInfo& info) { pthread_once(&once, init); AutoLock lock(*nodeArray); const std::string& node = info.node(); const std::string& type = info.name(); for (NodeArray::iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active() && type == (*k).type() && node == (*k).node()) { (*k).offLine(true); } } } void ClusterNodes::offLine(const std::string& host, int port) { pthread_once(&once, init); AutoLock lock(*nodeArray); for (NodeArray::iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active() && host == (*k).host() && port == (*k).port()) { (*k).offLine(true); } } } void ClusterNodes::onLine(const std::string& host, int port) { pthread_once(&once, init); AutoLock lock(*nodeArray); for (NodeArray::iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active() && host == (*k).host() && port == (*k).port()) { (*k).offLine(false); } } } void ClusterNodes::list(std::ostream& out) { pthread_once(&once, init); AutoLock lock(*nodeArray); for (NodeArray::const_iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active()) { out << *k << std::endl; } } } std::vector ClusterNodes::all() { pthread_once(&once, init); std::vector result; AutoLock lock(*nodeArray); for (NodeArray::const_iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active()) { result.push_back(k->asNodeInfo()); } } return result; } void ClusterNodes::json(JSON& j) { pthread_once(&once, init); j.startList(); AutoLock lock(*nodeArray); for (NodeArray::const_iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active()) { (*k).json(j); } } j.endList(); } void ClusterNodes::send(Stream& s) { pthread_once(&once, init); AutoLock lock(*nodeArray); for (NodeArray::const_iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { if ((*k).active()) { s << bool(true); (*k).send(s); } } s << bool(false); } void ClusterNodes::receive(Stream& s) { pthread_once(&once, init); AutoLock lock(*nodeArray); for (NodeArray::iterator k = nodeArray->begin(); k != nodeArray->end(); ++k) { (*k).active(false); } bool more; NodeArray::iterator k = nodeArray->begin(); for (;;) { s >> more; if (!more) { break; } ASSERT(k != nodeArray->end()); (*k).receive(s); ++k; } } bool ClusterNodes::lookUpHost(const std::string& type, const std::string& host, NodeInfo& result) { pthread_once(&once, init); AutoLock lock(*nodeArray); auto ip = eckit::net::IPAddress::hostAddress(host); for (const auto& k : *nodeArray) { if (k.active() && type == k.type() && ip == eckit::net::IPAddress::hostAddress(k.host())) { result = k.asNodeInfo(); return true; } } return false; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/cluster/ClusterNodes.h0000664000175000017500000000360515161702250021547 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File ClusterNodes.h // Baudouin Raoult - (c) ECMWF Jul 11 #ifndef eckit_ClusterNodes_h #define eckit_ClusterNodes_h #include "eckit/eckit.h" #include "eckit/io/cluster/NodeInfo.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class Stream; class JSON; class ClusterNodes { public: // -- Class methods // None static void reset(); static void cleanup(); static void forget(const NodeInfo&); static void offLine(const NodeInfo&); static void refresh(const NodeInfo&); static void list(std::ostream& out); static void json(JSON& out); static void send(Stream& s); static void receive(Stream& s); static NodeInfo lookUp(const std::string& type, const std::string& node); // Attributes argument is optional. Determines a labelled subset of a given type of node static NodeInfo any(const std::string& type, const std::set& attributes = {}); static bool available(const std::string&, const std::string&); static bool offsite(const std::string&, const std::string&); static void offLine(const std::string&, int); static void onLine(const std::string&, int); static std::vector all(); static bool lookUpHost(const std::string& type, const std::string& host, NodeInfo&); }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/cluster/ClusterNode.cc0000664000175000017500000000745115161702250021525 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/Resource.h" #include "eckit/io/cluster/ClusterDisks.h" #include "eckit/io/cluster/ClusterNode.h" #include "eckit/io/cluster/ClusterNodes.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/net/Port.h" #include "eckit/net/TCPClient.h" #include "eckit/net/TCPStream.h" #include "eckit/runtime/Monitor.h" #include "eckit/thread/Thread.h" #include "eckit/thread/ThreadControler.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class ClusterHeartBeat : public Thread { int hostOffset_; ClusterNode& owner_; virtual void run(); public: ClusterHeartBeat(ClusterNode& owner, int hostOffset = 0) : hostOffset_(hostOffset), owner_(owner) {} }; void ClusterHeartBeat::run() { Monitor::instance().name("heartbeat"); std::vector hosts = Resource>("clusterHost", {"localhost"}); int port = net::Port("cluster", 9555); std::string reply; NodeInfo remote; // Which cluster host are we talking to in this thread ASSERT(hostOffset_ < hosts.size()); std::string host = hosts[hostOffset_]; // If there are other cluster hosts to consider, spawn heartbeat threads for them too if (hosts.size() > hostOffset_ + 1) { ThreadControler t(new ClusterHeartBeat(owner_, hostOffset_ + 1)); t.start(); } for (;;) { net::TCPClient client; try { Log::status() << "Connecting to " << host << ":" << port << std::endl; net::TCPStream s(client.connect(host, port)); Log::status() << "Connected to " << host << ":" << port << std::endl; NodeInfo::thisNode().port(owner_.port()); NodeInfo::thisNode().attributes(owner_.attributes()); remote = NodeInfo::sendLogin(s); owner_.initialise(s); // s << owner_.port(); char x[] = ".:"; int n = 0; for (;;) { Monitor::instance().state(x[n]); n = 1 - n; s << "heartbeat"; s >> reply; if (reply == "sync") { ClusterNodes::receive(s); bool syncDisks; s >> syncDisks; if (syncDisks) { ClusterDisks::receive(s); } } if (reply == "exit") { ::exit(0); } ::sleep(20); owner_.refresh(s); } } catch (std::exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; Log::error() << "** Exception is handled" << std::endl; } } } //---------------------------------------------------------------------------------------------------------------------- ClusterNode::ClusterNode() {} ClusterNode::~ClusterNode() {} void ClusterNode::heartbeat() { ThreadControler t(new ClusterHeartBeat(*this)); t.start(); } // By default, there are no specified attributes const std::set& ClusterNode::attributes() const { static std::set nullAttrs; return nullAttrs; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/TCPSocketHandle.h0000664000175000017500000000426215161702250020367 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/TCPSocketHandle.h // Baudouin Raoult - ECMWF Jul 96 #ifndef eckit_filesystem_TCPSocketHandle_h #define eckit_filesystem_TCPSocketHandle_h #include "eckit/io/DataHandle.h" #include "eckit/net/TCPSocket.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- // Does not takes ownership of the socket // See net::TCPSocketHandle below class InstantTCPSocketHandle : public DataHandle { public: // -- Contructors InstantTCPSocketHandle(net::TCPSocket&); // -- Destructor ~InstantTCPSocketHandle(); // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void rewind() override; void print(std::ostream&) const override; Offset seek(const Offset&) override; bool canSeek() const override { return true; } // From Streamable // -- Class methods protected: // -- Members net::TCPSocket& connection_; private: // No copy allowed InstantTCPSocketHandle(const InstantTCPSocketHandle&); InstantTCPSocketHandle& operator=(const InstantTCPSocketHandle&); bool read_; Offset position_; // -- Class members }; // Takes ownership of the socket class TCPSocketHandle : public InstantTCPSocketHandle { public: TCPSocketHandle(net::TCPSocket&); private: net::TCPSocket socket_; void print(std::ostream&) const override; void close() override; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/Offset.cc0000664000175000017500000000474415161702250017045 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "Offset.h" #include "eckit/exception/Exceptions.h" #include "eckit/persist/DumpLoad.h" #include "eckit/serialisation/Stream.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- void sort(OffsetList& offset, LengthList& length) { ASSERT(offset.size() == length.size()); size_t i = 0; using OffsetLengthMap = std::map; OffsetLengthMap ol; for (i = 0; i < offset.size(); i++) { ol[offset[i]] = length[i]; } i = 0; for (OffsetLengthMap::iterator j = ol.begin(); j != ol.end(); ++j, ++i) { offset[i] = (*j).first; length[i] = (*j).second; } } bool compress(OffsetList& offset, LengthList& length) { ASSERT(offset.size() == length.size()); size_t j = 0; for (size_t i = 1; i < offset.size(); i++) { if ((offset[j] + length[j]) == offset[i]) { length[j] += length[i]; } else { ASSERT(++j < offset.size()); offset[j] = offset[i]; length[j] = length[i]; } } long save = offset.size() - j - 1; if (save > 0) { offset.erase(offset.begin() + j + 1, offset.end()); length.erase(length.begin() + j + 1, length.end()); } return save > 0; } void accumulate(const LengthList& length, OffsetList& offset, const Offset& from) { offset.clear(); offset.reserve(length.size()); Offset o(from); for (size_t i = 0; i < length.size(); i++) { offset.push_back(o); o += length[i]; } } std::ostream& operator<<(std::ostream& s, const Offset& x) { return s << x.value_; } Stream& operator<<(Stream& s, const Offset& x) { return s << x.value_; } Stream& operator>>(Stream& s, Offset& x) { s >> x.value_; return s; } void Offset::dump(DumpLoad& a) const { a.dump(value_); } void Offset::load(DumpLoad& a) { a.load(value_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/Pipeline.cc0000664000175000017500000000656315161702250017365 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/io/FileDescHandle.h" #include "eckit/io/Pipeline.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/log/Progress.h" #include "eckit/log/Timer.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/MutexCond.h" #include "eckit/thread/Thread.h" #include "eckit/thread/ThreadControler.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class PipelineReader : public Thread { Pipeline& owner_; DataHandle& in_; DataHandle& out_; public: PipelineReader(Pipeline& owner_, DataHandle& in, DataHandle& out); virtual void run(); }; PipelineReader::PipelineReader(Pipeline& owner, DataHandle& in, DataHandle& out) : owner_(owner), in_(in), out_(out) {} void PipelineReader::run() { try { in_.saveInto(out_, owner_.watcher_); } catch (std::exception& e) { owner_.error(e.what()); } } class PipelineExecutor : public Thread { Pipeline& owner_; DataHandle& in_; DataHandle& out_; public: PipelineExecutor(Pipeline& owner_, DataHandle& in, DataHandle& out); virtual void run(); }; PipelineExecutor::PipelineExecutor(Pipeline& owner, DataHandle& in, DataHandle& out) : owner_(owner), in_(in), out_(out) {} void PipelineExecutor::run() { try { owner_.execute(in_, out_); } catch (std::exception& e) { owner_.error(e.what()); } } Pipeline::Pipeline(TransferWatcher& watcher) : error_(false), watcher_(watcher) {} Pipeline::~Pipeline() {} inline void Pipeline::error(const std::string& why) { AutoLock lock(mutex_); error_ = true; why_ = why; } inline bool Pipeline::error() { AutoLock lock(mutex_); return error_; } Length Pipeline::copy(DataHandle& in, DataHandle& out) { int reader[2]; SYSCALL(::pipe(reader)); FileDescHandle reader_out(reader[1], true); FileDescHandle reader_in(reader[0], true); int process[2]; SYSCALL(::pipe(process)); FileDescHandle process_out(process[1], true); FileDescHandle process_in(process[0], true); ThreadControler thread1(new PipelineReader(*this, in, reader_out), false); thread1.start(); ThreadControler thread2(new PipelineExecutor(*this, reader_in, process_out), false); thread2.start(); Length total = 0; try { total = process_in.saveInto(out, watcher_); } catch (std::exception& e) { error(e.what()); } try { thread1.wait(); } catch (std::exception& e) { error(e.what()); } try { thread2.wait(); } catch (std::exception& e) { error(e.what()); } if (error_) { throw SeriousBug(why_); } return total; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/BufferedHandle.h0000664000175000017500000000422715161702250020313 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File io/BufferedHandle.h // Manuel Fuentes - ECMWF Jul 96 #ifndef eckit_filesystem_BufferedHandle_h #define eckit_filesystem_BufferedHandle_h #include "eckit/io/Buffer.h" #include "eckit/io/HandleHolder.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class BufferedHandle : public DataHandle, public HandleHolder { public: /// Contructor, taking ownership BufferedHandle(DataHandle*, size_t = 1024 * 1024, bool opened = false); /// Contructor, not taking ownership BufferedHandle(DataHandle&, size_t = 1024 * 1024, bool opened = false); /// Destructor ~BufferedHandle() override; // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void flush() override; void rewind() override; void print(std::ostream&) const override; void skip(const Length&) override; Offset seek(const Offset&) override; bool canSeek() const override { return handle().canSeek(); } Length estimate() override; Offset position() override; DataHandle* clone() const override; private: // methods void bufferFlush(); private: // members Buffer buffer_; size_t pos_; size_t size_; size_t used_; bool eof_; bool read_; Offset position_; bool opened_; std::string title() const override; void collectMetrics(const std::string& what) const override; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/BufferCache.h0000664000175000017500000000415015161702250017605 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File BufferCache.h // Baudouin Raoult - (c) ECMWF Jul 11 #ifndef eckit_BufferCache_h #define eckit_BufferCache_h #include #include #include #include #include "eckit/io/Buffer.h" namespace eckit { //----------------------------------------------------------------------------- class BufferCache { public: // -- Exceptions // None // -- Contructors BufferCache(size_t = 1024); BufferCache(const BufferCache&); BufferCache& operator=(const BufferCache&); // -- Destructor ~BufferCache(); // -- Convertors // None // -- Operators bool operator<(const BufferCache& other) const; // -- Methods // None void add(const void*, size_t); void reset(); size_t count() const { return count_; } const void* buffer() const { return buffer_; } time_t updated() const { return updated_; } // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed // -- Members size_t count_; Buffer buffer_; time_t updated_; // -- Methods // None // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends friend std::ostream& operator<<(std::ostream& s, const BufferCache& p) { p.print(s); return s; } }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/TCPHandle.cc0000664000175000017500000000500615161702250017351 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/TCPHandle.h" #include "eckit/io/MoverTransferSelection.h" #include "eckit/io/cluster/ClusterNodes.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ClassSpec TCPHandle::classSpec_ = { &DataHandle::classSpec(), "TCPHandle", }; Reanimator TCPHandle::reanimator_; void TCPHandle::print(std::ostream& s) const { s << "TCPHandle[host=" << host_ << ",port=" << port_ << ']'; } void TCPHandle::encode(Stream& s) const { DataHandle::encode(s); s << host_; s << port_; } TCPHandle::TCPHandle(Stream& s) : DataHandle(s), port_(0) { s >> host_; s >> port_; } TCPHandle::TCPHandle(const std::string& host, int port) : host_(host), port_(port) {} TCPHandle::~TCPHandle() {} Length TCPHandle::openForRead() { connection_.connect(host_, port_); return 0; } void TCPHandle::openForWrite(const Length&) { connection_.connect(host_, port_); } void TCPHandle::openForAppend(const Length&) { NOTIMP; } long TCPHandle::read(void* buffer, long length) { return connection_.read(buffer, length); } long TCPHandle::write(const void* buffer, long length) { return connection_.write(buffer, length); } void TCPHandle::close() { connection_.close(); } void TCPHandle::rewind() { NOTIMP; } DataHandle* TCPHandle::clone() const { return new TCPHandle(host_, port_); } std::string TCPHandle::title() const { std::ostringstream os; os << "TCP[" << host_ << ":" << port_ << "]"; return os.str(); } void TCPHandle::selectMover(eckit::MoverTransferSelection& selection, bool read) const { // If we use a callback server that is collocated on a mover node // we want to use that mover NodeInfo node; if (ClusterNodes::lookUpHost("mover", host_, node)) { selection.preferredMover(node); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/Buffer.cc0000664000175000017500000000536315161702250017026 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/Buffer.h" #include #include #include "eckit/exception/Exceptions.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- namespace { char* allocate(size_t size) { return size == 0 ? nullptr : new char[size]; } void deallocate(char*& buffer) { delete[] buffer; buffer = nullptr; } } // namespace //---------------------------------------------------------------------------------------------------------------------- Buffer::Buffer(size_t size) : size_{size} { create(); } Buffer::Buffer(const void* p, size_t size) : size_{size} { create(); copy(p, size); } Buffer::Buffer(const std::string& s) : size_{s.length() + 1} { create(); copy(s); } Buffer::Buffer(Buffer&& rhs) noexcept : buffer_{rhs.buffer_}, size_{rhs.size_} { rhs.buffer_ = nullptr; rhs.size_ = 0; } Buffer& Buffer::operator=(Buffer&& rhs) noexcept { if (this == &rhs) { return *this; } deallocate(buffer_); buffer_ = rhs.buffer_; size_ = rhs.size_; rhs.buffer_ = nullptr; rhs.size_ = 0; return *this; } Buffer::~Buffer() { destroy(); } void Buffer::zero() { if (buffer_ != nullptr) { std::memset(buffer_, 0, size_); } } void Buffer::create() { buffer_ = allocate(size_); } void Buffer::destroy() { deallocate(buffer_); size_ = 0; } void Buffer::copy(const std::string& s) { if (buffer_ != nullptr) { std::strncpy(buffer_, s.c_str(), std::min(size_, s.size() + 1)); } } void Buffer::copy(const void* p, size_t size, size_t pos) { ASSERT(size_ >= pos + size); if (buffer_ != nullptr && size > 0) { std::memcpy(buffer_ + pos, p, size); } } void Buffer::resize(size_t size, bool preserveData) { if (size != size_) { if (preserveData) { auto* newbuffer = allocate(size); std::memcpy(newbuffer, buffer_, std::min(size_, size)); deallocate(buffer_); size_ = size; buffer_ = newbuffer; } else { deallocate(buffer_); size_ = size; buffer_ = allocate(size); } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/io/FileLocker.h0000664000175000017500000000401615161702250017470 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Feb 2002 #ifndef eckit_FileLocker_h #define eckit_FileLocker_h #include "sys/types.h" // for off_t #include "eckit/eckit.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class FileLocker { public: // -- Exceptions // None // -- Contructors FileLocker(int fd); // -- Destructor ~FileLocker(); // -- Convertors // None // -- Operators // None // -- Methods void lockShared(off_t off = 0, off_t len = 0); void lockExclusive(off_t off = 0, off_t len = 0); void unlock(off_t off = 0, off_t len = 0); // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods // void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed FileLocker(const FileLocker&); FileLocker& operator=(const FileLocker&); // -- Members int fd_; // -- Methods // None void lockRange(off_t, off_t, int cmd, int type); // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FileLocker& p) // { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/PartHandle.h0000664000175000017500000000456615161702250017505 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Partio/FileHandle.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_filesystem_PartHandle_h #define eckit_filesystem_PartHandle_h #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/io/HandleHolder.h" #include "eckit/filesystem/PathName.h" #include "eckit/types/Types.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class PartHandle : public DataHandle, public HandleHolder { public: // -- Contructors PartHandle(DataHandle& handle, const OffsetList&, const LengthList&); PartHandle(DataHandle& handle, const Offset&, const Length&); PartHandle(DataHandle* handle, const OffsetList&, const LengthList&); PartHandle(DataHandle* handle, const Offset&, const Length&); PartHandle(Stream&); // -- Destructor ~PartHandle(); // -- Methods // -- Overridden methods // From DataHandle Length openForRead() override; void openForWrite(const Length&) override; void openForAppend(const Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; void rewind() override; void print(std::ostream&) const override; Length estimate() override; void restartReadFrom(const Offset& from) override; // From Streamable void encode(Stream&) const override; const ReanimatorBase& reanimator() const override { return reanimator_; } // -- Class methods static const ClassSpec& classSpec() { return classSpec_; } private: // -- Members long long pos_; Ordinal index_; OffsetList offset_; LengthList length_; // -- Methods long read1(char*, long); // -- Class members static ClassSpec classSpec_; static Reanimator reanimator_; }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/io/SockBuf.h0000664000175000017500000000210315161702250017000 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SockBuf.h // Baudouin Raoult - ECMWF Mar 97 #ifndef eckit_SockBuf_h #define eckit_SockBuf_h #include "eckit/net/TCPSocket.h" namespace eckit { class SockBuf : public std::streambuf { public: // -- Contructors SockBuf(net::TCPSocket& proto); // -- Destructor ~SockBuf(); private: // No copy allowed SockBuf(const SockBuf&); SockBuf& operator=(const SockBuf&); // -- Members char in_[1]; char out_[80]; net::TCPSocket& protocol_; // -- Overridden methods // From streambuf virtual int overflow(int c); virtual int underflow(); virtual int sync(); // virtual int uflow(); }; } // namespace eckit #endif eckit-2.0.7/src/eckit/io/PooledFileDescriptor.cc0000664000175000017500000000425015161702250021670 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/io/FDataSync.h" #include "eckit/io/PooledFileDescriptor.h" namespace eckit { PooledFileDescriptor::PooledFileDescriptor(const PathName& path, bool readOnly) : path_(path), fd_(-1), readOnly_(readOnly) {} PooledFileDescriptor::~PooledFileDescriptor() { close(); } void PooledFileDescriptor::open() { if (readOnly_) { file_.reset(new PooledFile(path_)); file_->open(); fd_ = file_->fileno(); } else { SYSCALL2(fd_ = ::open(path_.localPath(), O_RDWR | O_CREAT, 0777), path_); } } void PooledFileDescriptor::close() { if (fd_ < 0) { return; } if (readOnly_) { ASSERT(file_); file_->close(); file_.reset(); } else { SYSCALL(::close(fd_)); } fd_ = -1; } ssize_t PooledFileDescriptor::read(void* buf, size_t nbyte) { if (readOnly_) { ASSERT(file_); return file_->read(buf, nbyte); } ssize_t len; SYSCALL(len = ::read(fd_, buf, nbyte)); return len; } ssize_t PooledFileDescriptor::write(const void* buf, size_t nbyte) { ASSERT(!readOnly_); ssize_t len; SYSCALL(len = ::write(fd_, buf, nbyte)); return len; } void PooledFileDescriptor::sync() { ASSERT(!readOnly_); SYSCALL2(eckit::fdatasync(fd_), path_); } off_t PooledFileDescriptor::seek(off_t offset) { if (readOnly_) { ASSERT(file_); return file_->seek(offset); } off_t here; SYSCALL(here = ::lseek(fd_, offset, SEEK_SET)); return here; } off_t PooledFileDescriptor::seekEnd() { if (readOnly_) { ASSERT(file_); return file_->seekEnd(); } off_t here; SYSCALL(here = ::lseek(fd_, 0, SEEK_END)); return here; } } // namespace eckit eckit-2.0.7/src/eckit/codec/0000775000175000017500000000000015161702250015745 5ustar alastairalastaireckit-2.0.7/src/eckit/codec/RecordWriter.cc0000664000175000017500000002064515161702250020676 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/RecordWriter.h" #include "eckit/codec/Exceptions.h" #include "eckit/codec/detail/Checksum.h" #include "eckit/codec/detail/Defaults.h" #include "eckit/codec/detail/Encoder.h" #include "eckit/codec/detail/RecordSections.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- template inline void write_struct(OStream& out, const Struct& s) { static_assert(Struct::bytes == sizeof(Struct)); if (out.write(reinterpret_cast(&s), sizeof(s)) != sizeof(s)) { throw WriteError("Could not write struct to stream"); } } //--------------------------------------------------------------------------------------------------------------------- template inline void write_string(OStream& out, const std::string& s) { if (out.write(s.data(), s.size()) != s.size()) { throw WriteError("Could not write string to stream"); } } //--------------------------------------------------------------------------------------------------------------------- size_t RecordWriter::write(Stream out) const { RecordHead r; auto begin_of_record = out.position(); auto position = [&begin_of_record, &out]() { return out.position() - begin_of_record; }; std::vector index; // Begin Record // ------------ write_struct(out, r); // Metadata section // ---------------- { r.metadata_offset = position(); write_struct(out, RecordMetadataSection::Begin()); auto metadata_str = metadata(); write_string(out, metadata_str); write_struct(out, RecordMetadataSection::End()); r.metadata_length = position() - r.metadata_offset; r.metadata_checksum = do_checksum_ != 0 ? codec::checksum(metadata_str.data(), metadata_str.size()) : std::string("none:"); // Index section // ------------- auto nb_data_sections = static_cast(nb_data_sections_); r.index_offset = position(); write_struct(out, RecordDataIndexSection::Begin()); index.resize(nb_data_sections); for (size_t i = 0; i < nb_data_sections; ++i) { write_struct(out, index[i]); } write_struct(out, RecordDataIndexSection::End()); r.index_length = position() - r.index_offset; } // Data sections // ------------- { size_t i{0}; for (const auto& key : keys_) { const auto& encoder = encoders_.at(key); const auto& info = info_.at(key); if (info.section() == 0) { continue; } Data data; encode_data(encoder, data); data.compress(info.compression()); auto& data_section = index[i]; data_section.offset = position(); write_struct(out, RecordDataSection::Begin()); if (data.write(out) != data.size()) { throw WriteError("Could not write data for item " + key + " to stream"); } write_struct(out, RecordDataSection::End()); data_section.length = position() - data_section.offset; data_section.checksum = do_checksum_ != 0 ? data.checksum() : std::string("none:"); ++i; } } // End Record // ---------- write_struct(out, RecordEnd()); auto end_of_record = out.position(); r.record_length = end_of_record - begin_of_record; r.time = Time::now(); out.seek(begin_of_record); write_struct(out, r); out.seek(begin_of_record + r.index_offset + sizeof(RecordDataIndexSection::Begin)); for (auto& entry : index) { write_struct(out, entry); } out.seek(end_of_record); // So that following writes will not overwrite return r.record_length; } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::compression(const std::string& c) { compression_ = c; } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::compression(bool on) { if (on) { compression_ = defaults::compression_algorithm(); // still possible to be "none" } else { compression_ = "none"; } } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::checksum(bool on) { do_checksum_ = on && defaults::checksum_write() ? 1 : 0; // possible to be off via environment } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::set(const RecordWriter::Key& key, Link&& link, const Configuration&) { keys_.emplace_back(key); encoders_[key] = Encoder{link}; info_.emplace(key, DataInfo{}); } //--------------------------------------------------------------------------------------------------------------------- void RecordWriter::set(const RecordWriter::Key& key, Encoder&& encoder, const Configuration& config) { DataInfo info; if (encoder.encodes_data()) { ++nb_data_sections_; info.compression(config.getString("compression", compression_)); info.section(nb_data_sections_); } keys_.emplace_back(key); encoders_[key] = std::move(encoder); info_.emplace(key, std::move(info)); } //--------------------------------------------------------------------------------------------------------------------- size_t RecordWriter::write(const PathName& path, Mode mode) const { return write(OutputFileStream(path, mode)); } //--------------------------------------------------------------------------------------------------------------------- size_t RecordWriter::write(DataHandle& out) const { return write(Stream(out)); } //--------------------------------------------------------------------------------------------------------------------- size_t RecordWriter::estimateMaximumSize() const { size_t size{0}; size += sizeof(RecordHead); size += sizeof(RecordMetadataSection::Begin); size += metadata().size(); size += sizeof(RecordMetadataSection::End); size += sizeof(RecordDataIndexSection::Begin); size += static_cast(nb_data_sections_) * sizeof(RecordDataIndexSection::Entry); size += sizeof(RecordDataIndexSection::End); for (const auto& key : keys_) { const auto& encoder = encoders_.at(key); const auto& info = info_.at(key); if (info.section() == 0) { continue; } size += sizeof(RecordDataSection::Begin); { Metadata m; size_t max_data_size = encode_metadata(encoder, m); if (info.compression() != "none") { max_data_size = static_cast(1.2 * static_cast(max_data_size)); max_data_size = std::max(max_data_size, 10 * 1024); // minimum 10KB } size += max_data_size; } size += sizeof(RecordDataSection::End); } size += sizeof(RecordEnd); return size; } //--------------------------------------------------------------------------------------------------------------------- std::string RecordWriter::metadata() const { Metadata metadata; for (const auto& key : keys_) { const auto& encoder = encoders_.at(key); const auto& info = info_.at(key); Metadata m; encode_metadata(encoder, m); if (info.section() != 0) { m.set("data.section", info.section()); if (info.compression() != "none") { m.set("data.compression.type", info.compression()); } } metadata.set(key, m); } std::stringstream ss; codec::write(metadata, ss); return ss.str(); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Record.h0000664000175000017500000000332215161702250017334 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include "eckit/codec/detail/Endian.h" #include "eckit/codec/detail/Time.h" #include "eckit/codec/detail/Version.h" namespace eckit::codec { class Stream; class ParsedRecord; class Metadata; //--------------------------------------------------------------------------------------------------------------------- class Record { public: struct URI { std::string str() const; std::string path; std::uint64_t offset; URI() = default; URI(const URI&) = default; explicit URI(const std::string& _path, std::uint64_t _offset = 0) : path(_path), offset(_offset) {} }; private: std::shared_ptr record_; public: Record(); bool empty() const; Record& read(Stream&, bool read_to_end = false); const Metadata& metadata(const std::string& key) const; Endian endian() const; Version version() const; Time time() const; std::uint64_t size() const; const std::vector& keys() const; bool has(const std::string& key); explicit operator const ParsedRecord&() const; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/RecordReader.h0000664000175000017500000000422115161702250020456 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/Metadata.h" #include "eckit/codec/ReadRequest.h" #include "eckit/codec/Session.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class RecordReader { public: explicit RecordReader(const Record::URI& ref); explicit RecordReader(const std::string& path, std::uint64_t offset = 0); explicit RecordReader(Stream stream, std::uint64_t offset = 0); template ReadRequest& read(const std::string& key, T& value) { trace("read(" + key + ")", __FILE__, __LINE__, __func__); if (stream_) { trace("stream", __FILE__, __LINE__, __func__); requests_.emplace(key, ReadRequest{stream_, offset_, key, value}); } else { requests_.emplace(key, ReadRequest{uri(key), value}); } if (do_checksum_ >= 0) { requests_.at(key).checksum(do_checksum_); } return requests_.at(key); } void wait(const std::string& key); void wait(); ReadRequest& request(const std::string& key); Metadata metadata(const std::string& key); void checksum(bool); private: Record::URI uri() const; RecordItem::URI uri(const std::string& key) const; void trace(const std::string&, const char* file, int line, const char* func); private: Session session_; Stream stream_; std::map requests_; std::string path_; std::uint64_t offset_; int do_checksum_{-1}; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/RecordItem.cc0000664000175000017500000001062715161702250020317 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/RecordItem.h" #include "eckit/codec/Exceptions.h" #include "eckit/filesystem/URI.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- RecordItem::URI::URI(const std::string& _uri) { eckit::URI uri{_uri}; ASSERT(uri.scheme() == "file"); ASSERT(not uri.query("key").empty()); path = uri.path(); offset = 0; if (not uri.query("offset").empty()) { offset = std::stoul(uri.query("offset")); } key = uri.query("key"); } //--------------------------------------------------------------------------------------------------------------------- RecordItem::URI::URI(const std::string& _path, uint64_t _offset, const std::string& _key) : path(_path), offset(_offset), key(_key) {} //--------------------------------------------------------------------------------------------------------------------- std::string RecordItem::URI::str() const { eckit::URI uri("file", PathName(path)); uri.query("offset", std::to_string(offset)); uri.query("key", key); return uri.asRawString(); } //--------------------------------------------------------------------------------------------------------------------- RecordItem::RecordItem(RecordItem&& other) : metadata_(std::move(other.metadata_)), data_(std::move(other.data_)) {} //--------------------------------------------------------------------------------------------------------------------- RecordItem::RecordItem(Metadata&& metadata, Data&& data) : metadata_(new Metadata(metadata)), data_(std::move(data)) {} //--------------------------------------------------------------------------------------------------------------------- const Data& RecordItem::data() const { return data_; } //--------------------------------------------------------------------------------------------------------------------- const Metadata& RecordItem::metadata() const { return *metadata_; } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::metadata(const Metadata& m) { *metadata_ = m; } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::data(Data&& d) { data_ = std::move(d); } //--------------------------------------------------------------------------------------------------------------------- bool RecordItem::empty() const { return metadata().empty(); } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::clear() { data_.clear(); metadata_.reset(new Metadata()); } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::decompress() { ASSERT(not empty()); if (metadata().data.compressed()) { data_.decompress(metadata().data.compression(), metadata().data.size()); } metadata_->data.compressed(false); } //--------------------------------------------------------------------------------------------------------------------- void RecordItem::compress() { ASSERT(not empty()); if (not metadata().data.compressed() && metadata().data.compression() != "none") { data_.compress(metadata().data.compression()); metadata_->data.compressed(true); } } //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const RecordItem& in, Metadata& metadata) { metadata.set(in.metadata()); return in.data().size(); } //--------------------------------------------------------------------------------------------------------------------- void encode_data(const RecordItem& in, Data& out) { out.assign(in.data()); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/ReadRequest.h0000664000175000017500000000360515161702250020346 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/RecordItem.h" #include "eckit/codec/detail/Decoder.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class ReadRequest { public: ReadRequest(ReadRequest&& other); template ReadRequest(Stream stream, size_t offset, const std::string& key, T& value) : ReadRequest{stream, offset, key, new Decoder(value)} {} template ReadRequest(const std::string& URI, T& value) : ReadRequest{URI, new Decoder(value)} {} template ReadRequest(const RecordItem::URI& URI, T& value) : ReadRequest{URI.str(), value} {} ReadRequest() = delete; ReadRequest(const ReadRequest&) = delete; ~ReadRequest(); void read(); void checksum(); void decompress(); void decode(); void wait(); void checksum(bool); private: ReadRequest(const std::string& URI, Decoder* decoder); ReadRequest(Stream, size_t offset, const std::string& key, Decoder*); Stream stream_; size_t offset_; std::string key_; std::string uri_; std::unique_ptr decoder_; std::unique_ptr item_; bool do_checksum_{true}; bool finished_{false}; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/RecordReader.cc0000664000175000017500000000572215161702250020623 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/RecordReader.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/RecordItemReader.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- RecordReader::RecordReader(const Record::URI& ref) : RecordReader(ref.path, ref.offset) {} //--------------------------------------------------------------------------------------------------------------------- RecordReader::RecordReader(const std::string& path, uint64_t offset) : path_{path}, offset_{offset} {} RecordReader::RecordReader(Stream stream, uint64_t offset) : stream_{stream}, offset_{offset} {} //--------------------------------------------------------------------------------------------------------------------- Record::URI RecordReader::uri() const { Record::URI _uri; _uri.path = path_; _uri.offset = offset_; return _uri; } //--------------------------------------------------------------------------------------------------------------------- RecordItem::URI RecordReader::uri(const std::string& key) const { return {path_, offset_, key}; } //--------------------------------------------------------------------------------------------------------------------- void RecordReader::trace(const std::string&, const char* file, int line, const char* func) {} //--------------------------------------------------------------------------------------------------------------------- void RecordReader::wait(const std::string& key) { request(key).wait(); } //--------------------------------------------------------------------------------------------------------------------- void RecordReader::wait() { // This can be optimized perhaps to overlap IO with decoding in multithreaded environment for (auto& pair : requests_) { auto& request = pair.second; request.wait(); } } //--------------------------------------------------------------------------------------------------------------------- ReadRequest& RecordReader::request(const std::string& key) { return requests_.at(key); } //--------------------------------------------------------------------------------------------------------------------- Metadata RecordReader::metadata(const std::string& key) { Metadata metadata; RecordItemReader{uri(key)}.read(metadata); return metadata; } void RecordReader::checksum(bool b) { do_checksum_ = b ? 1 : 0; } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/codec.h0000664000175000017500000001060315161702250017173 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/Exceptions.h" #include "eckit/codec/FileStream.h" #include "eckit/codec/Record.h" #include "eckit/codec/RecordItemReader.h" #include "eckit/codec/RecordPrinter.h" #include "eckit/codec/RecordReader.h" #include "eckit/codec/RecordWriter.h" #include "eckit/codec/Session.h" #include "eckit/codec/Stream.h" #include "eckit/codec/detail/Link.h" #include "eckit/codec/detail/Reference.h" #include "eckit/codec/detail/StaticAssert.h" #include "eckit/codec/detail/sfinae.h" #include "eckit/codec/types/array.h" #include "eckit/codec/types/scalar.h" #include "eckit/codec/types/string.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- inline Link link(const std::string& uri) { return Link{uri}; } //--------------------------------------------------------------------------------------------------------------------- template = 0> Reference ref(const T& x, tag::enable_static_assert = tag::enable_static_assert()) { static_assert(is_encodable(), "in ref(const Value&)" "\n" "\n Static assertion failed" "\n -----------------------" "\n" "\n Cannot encode values of referenced type." "\n" "\n Implement the functions" "\n" "\n void encode_data(const Value& in, Data& out);" "\n size_t encode_metadata(const Value& value, Metadata& metadata);" "\n" "\n or alternatively a conversion function to types::ArrayView" "\n" "\n void interprete(const Value& in, types::ArrayView& out)" "\n" "\n Rules of argument-dependent-lookup apply." "\n --> Functions need to be declared in namespace of any of the arguments." "\n" "\n Note, turn this into a runtime exception by calling this function instead:" "\n" "\n ref(const T&, no_static_assert() )" "\n"); return Reference(x); } template = 0> Reference ref(const T& x, tag::disable_static_assert) { if (not is_encodable()) { throw NotEncodable(x); } return Reference(x); } template = 0> ArrayReference ref(const T& x, tag::enable_static_assert = tag::enable_static_assert()) { ArrayReference w; interprete(x, w); return w; } //--------------------------------------------------------------------------------------------------------------------- template RecordItem copy(T&& value, tag::disable_static_assert) { return RecordItem(std::forward(value), tag::disable_static_assert()); } template RecordItem copy(T&& value) { return RecordItem(std::forward(value)); } //--------------------------------------------------------------------------------------------------------------------- template void encode(const T& in, Metadata& metadata, Data& data, tag::enable_static_assert = tag::enable_static_assert()) { auto referenced = ref(in, tag::enable_static_assert()); sfinae::encode_metadata(referenced, metadata); sfinae::encode_data(referenced, data); } template void encode(const T& in, Metadata& metadata, Data& data, tag::disable_static_assert) { auto referenced = ref(in, tag::disable_static_assert()); sfinae::encode_metadata(referenced, metadata); sfinae::encode_data(referenced, data); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Metadata.h0000664000175000017500000000466715161702250017653 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/codec/Stream.h" #include "eckit/codec/detail/Checksum.h" #include "eckit/codec/detail/DataInfo.h" #include "eckit/codec/detail/Endian.h" #include "eckit/codec/detail/Link.h" #include "eckit/codec/detail/RecordInfo.h" #include "eckit/codec/detail/Type.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/value/Value.h" namespace eckit::codec { class Metadata; class Stream; //--------------------------------------------------------------------------------------------------------------------- size_t uncompressed_size(const Metadata& m); //--------------------------------------------------------------------------------------------------------------------- class Metadata : public LocalConfiguration { public: using LocalConfiguration::LocalConfiguration; Metadata() = default; Link link() const { return Link{getString("link", "")}; } Type type() const { return Type{getString("type", "")}; } void link(Metadata&&); std::string json() const; DataInfo data; RecordInfo record; // extended LocalConfiguration: using LocalConfiguration::set; Metadata& set(const Configuration& other) { LocalConfiguration::set(other); return *this; } /// @brief Constructor immediately setting a value. template Metadata(const std::string& name, const ValueT& value) { set(name, value); } /// @brief Constructor starting from a Configuration Metadata(const Configuration& other) : LocalConfiguration(other) {} Metadata& remove(const std::string& name) { LocalConfiguration::remove(name); return *this; } }; //--------------------------------------------------------------------------------------------------------------------- void write(const Metadata&, std::ostream& out); void write(const Metadata&, Stream& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Stream.cc0000664000175000017500000000332615161702250017513 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/Stream.h" #include "eckit/codec/Exceptions.h" #include "eckit/io/DataHandle.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- Stream::Stream(DataHandle& datahandle) : ptr_(&datahandle) {} Stream::Stream(DataHandle* datahandle) : shared_(datahandle), ptr_(shared_.get()) {} Stream::Stream(std::shared_ptr datahandle) : shared_(datahandle), ptr_(shared_.get()) {} DataHandle& Stream::datahandle() { ASSERT(ptr_ != nullptr); return *ptr_; } uint64_t Stream::seek(uint64_t offset) { ASSERT(ptr_ != nullptr); return static_cast(ptr_->seek(static_cast(offset))); } uint64_t Stream::position() { ASSERT(ptr_ != nullptr); return static_cast(ptr_->position()); } uint64_t Stream::write(const void* data, size_t length) { ASSERT(ptr_ != nullptr); return static_cast(ptr_->write(data, static_cast(length))); } uint64_t Stream::read(void* data, size_t length) { ASSERT(ptr_ != nullptr); return static_cast(ptr_->read(data, static_cast(length))); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Data.cc0000664000175000017500000000635015161702250017131 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/Data.h" #include #include "eckit/codec/Exceptions.h" #include "eckit/codec/Stream.h" #include "eckit/codec/detail/Checksum.h" #include "eckit/utils/Compressor.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- Data::Data(void* p, size_t size) : buffer_(p, size), size_(size) {} std::uint64_t Data::write(Stream& out) const { if (size() > 0) { ASSERT(buffer_.size() >= size()); return out.write(buffer_.data(), size()); } return 0; } std::uint64_t Data::read(Stream& in, size_t size) { if (size > size_) { buffer_.resize(size); size_ = size; } return in.read(buffer_, size); } void Data::compress(const std::string& compression) { if (size_ > 0) { auto compressor = std::unique_ptr(CompressorFactory::instance().build(compression)); if (dynamic_cast(compressor.get()) != nullptr) { return; } Buffer compressed(static_cast(1.2 * static_cast(size_))); size_ = compressor->compress(buffer_, size_, compressed); buffer_ = std::move(compressed); } } void Data::decompress(const std::string& compression, size_t uncompressed_size) { auto compressor = std::unique_ptr(CompressorFactory::instance().build(compression)); if (dynamic_cast(compressor.get()) != nullptr) { return; } Buffer uncompressed(static_cast(1.2 * static_cast(uncompressed_size))); compressor->uncompress(buffer_, size_, uncompressed, uncompressed_size); size_ = uncompressed_size; buffer_ = std::move(uncompressed); } void Data::clear() { buffer_ = Buffer{}; size_ = 0; } std::string Data::checksum(const std::string& algorithm) const { return codec::checksum(buffer_, size_, algorithm); } void Data::assign(const Data& other) { if (other.size() > buffer_.size()) { buffer_.resize(other.size()); } size_ = other.size(); buffer_.copy(other.buffer_, size_); } void Data::assign(const void* p, size_t s) { if (s > size()) { buffer_.resize(s); } size_ = s; buffer_.copy(p, size_); } Data& compress(Data& data, const std::string& compression) { data.compress(compression); return data; } Data& decompress(Data& data, const std::string& compression, size_t uncompressed_size) { data.decompress(compression, uncompressed_size); return data; } //--------------------------------------------------------------------------------------------------------------------- void encode(const Data& in, Data& out) { out.assign(in); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Session.h0000664000175000017500000000203715161702250017543 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/codec/Record.h" #include "eckit/codec/Stream.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class Session { public: Session(); ~Session(); static bool active(); static Record record(const std::string& path, size_t offset); static Record record(Stream, size_t offset); static void store(Stream stream); }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/FileStream.cc0000664000175000017500000001136015161702250020310 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/FileStream.h" #include "eckit/codec/Session.h" #include "eckit/io/FileHandle.h" #include "eckit/io/PooledHandle.h" namespace eckit::codec { namespace { //--------------------------------------------------------------------------------------------------------------------- /// DataHandle that implements file IO for reading and writing /// Main differences with eckit::FileHandle: /// - Automatic opening and closing of file /// - Openmode argument: /// * read: for reading /// * write: for writing, will overwrite existing file /// * append: for appending implemented via write and seek to eof. class FileHandle : public eckit::FileHandle { public: FileHandle(const PathName& path, char openmode) : eckit::FileHandle(path, openmode == 'a' /*overwrite*/) { if (openmode == 'r') { openForRead(); } else if (openmode == 'w' || (openmode == 'a' && not path.exists())) { openForWrite(0); } else if (openmode == 'a') { openForWrite(path.size()); seek(Offset(path.size())); } } void close() override { if (not closed_) { eckit::FileHandle::close(); closed_ = true; } } FileHandle(const PathName& path, Mode openmode) : FileHandle(path, openmode == Mode::read ? 'r' : openmode == Mode::write ? 'w' : 'a') {} FileHandle(const PathName& path, const std::string& openmode) : FileHandle(path, openmode[0]) {} FileHandle(const FileHandle&) = delete; FileHandle(FileHandle&&) = delete; ~FileHandle() override { close(); } void operator=(const FileHandle&) = delete; void operator=(FileHandle&&) = delete; private: bool closed_{false}; }; //--------------------------------------------------------------------------------------------------------------------- /// DataHandle that implements file reading only. /// Internally there is a registry of opened files which avoids /// opening the same file multiple times. /// Note that close() will not actually close the file when there /// is another PooledHandle referencing the same file. /// /// Main difference with eckit::PooledHandle /// - Automatic opening and closing of file class PooledHandle : public eckit::PooledHandle { public: explicit PooledHandle(const PathName& path) : eckit::PooledHandle(path), path_(path) { openForRead(); } PooledHandle(const PooledHandle&) = delete; PooledHandle(PooledHandle&&) = delete; ~PooledHandle() override { close(); } void operator=(const PooledHandle&) = delete; void operator=(PooledHandle&&) = delete; PathName path_; }; } // namespace //--------------------------------------------------------------------------------------------------------------------- FileStream::FileStream(const PathName& path, char openmode) : Stream(openmode == 'r' ? static_cast(new PooledHandle(path)) : new FileHandle(path, openmode)) { if (openmode == 'r') { // Keep the PooledHandle alive until the end of active session Session::store(*this); } } FileStream::FileStream(const PathName& path, Mode openmode) : FileStream(path, openmode == Mode::read ? 'r' : openmode == Mode::write ? 'w' : 'a') {} FileStream::FileStream(const PathName& path, const std::string& openmode) : FileStream(path, openmode[0]) {} //--------------------------------------------------------------------------------------------------------------------- InputFileStream::InputFileStream(const PathName& path) : FileStream(path, Mode::read) {} //--------------------------------------------------------------------------------------------------------------------- OutputFileStream::OutputFileStream(const PathName& path, Mode openmode) : FileStream(path, openmode) {} OutputFileStream::OutputFileStream(const PathName& path, const std::string& openmode) : FileStream(path, openmode) {} OutputFileStream::OutputFileStream(const PathName& path, char openmode) : FileStream(path, openmode) {} void OutputFileStream::close() { datahandle().close(); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/types/0000775000175000017500000000000015161702250017111 5ustar alastairalastaireckit-2.0.7/src/eckit/codec/types/array.h0000664000175000017500000000122015161702250020373 0ustar alastairalastair/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/codec/types/array/ArrayReference.h" #include "eckit/codec/types/array/adaptors/StdArrayAdaptor.h" #include "eckit/codec/types/array/adaptors/StdVectorAdaptor.h" #include "eckit/codec/types/array/adaptors/StdVectorOfStdArrayAdaptor.h" eckit-2.0.7/src/eckit/codec/types/scalar.cc0000664000175000017500000001223615161702250020671 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // Cray C++ compiler should not try to optimize this code #ifdef _CRAYC #pragma _CRI noopt #endif // GNU C++ compiler (version 11) should not try to optimize this code #if defined(__GNUC__) && !defined(__NVCOMPILER) && !defined(__clang__) #pragma GCC optimize("O0") #endif #include "eckit/codec/types/scalar.h" #include #include #include "eckit/codec/Data.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/detail/Base64.h" #include "eckit/codec/detail/DataType.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/ByteSwap.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- template void decode_scalar(const Metadata& metadata, T& value) { ASSERT(metadata.getString("type") == "scalar"); ASSERT(metadata.getString("datatype") == DataType::str()); metadata.get("value", value); } template void decode_scalar_b64(const Metadata& metadata, T& value) { ASSERT(metadata.getString("type") == "scalar"); ASSERT(metadata.getString("datatype") == DataType::str()); std::string base64 = metadata.getString("base64"); T value_ns = Base64::decode(base64); if (Endian::native == Endian::little) { T tmp = value_ns; eckit::byteswap(tmp); value_ns = tmp; } value = value_ns; } //--------------------------------------------------------------------------------------------------------------------- template void encode_scalar_metadata(const T& value, Metadata& out) { out.set("type", "scalar"); out.set("datatype", DataType::str()); out.set("value", value); } inline void encode_scalar_metadata(const unsigned long& value, Metadata& out) { out.set("type", "scalar"); out.set("datatype", DataType::str()); out.set("value", static_cast(value)); } inline void encode_scalar_metadata(const unsigned long long& value, Metadata& out) { out.set("type", "scalar"); out.set("datatype", DataType::str()); out.set("value", static_cast(value)); } template size_t encode_scalar_metadata_b64(const T& value, Metadata& out) { encode_scalar_metadata(value, out); T value_ns = value; if (Endian::native == Endian::little) { eckit::byteswap(value_ns); } out.set("base64", Base64::encode(value_ns)); return 0; } //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const int& value, Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const long& value, Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const long long& value, Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const unsigned long& value, Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const unsigned long long& value, Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const float& value, Metadata& out) { return encode_scalar_metadata_b64(value, out); } size_t encode_metadata(const double& value, Metadata& out) { return encode_scalar_metadata_b64(value, out); } //--------------------------------------------------------------------------------------------------------------------- void encode_data(const int&, Data&) {} void encode_data(const long&, Data&) {} void encode_data(const long long&, Data&) {} void encode_data(const unsigned long&, Data&) {} void encode_data(const unsigned long long&, Data&) {} void encode_data(const float&, Data&) {} void encode_data(const double&, Data&) {} //--------------------------------------------------------------------------------------------------------------------- void decode(const Metadata& metadata, const Data&, int& value) { decode_scalar(metadata, value); } void decode(const Metadata& metadata, const Data&, long& value) { decode_scalar(metadata, value); } void decode(const Metadata& metadata, const Data&, long long& value) { decode_scalar(metadata, value); } void decode(const Metadata& metadata, const Data&, unsigned long& value) { decode_scalar(metadata, value); } void decode(const Metadata& metadata, const Data&, unsigned long long& value) { decode_scalar_b64(metadata, value); } void decode(const Metadata& metadata, const Data&, float& value) { decode_scalar_b64(metadata, value); } void decode(const Metadata& metadata, const Data&, double& value) { decode_scalar_b64(metadata, value); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/types/string.h0000664000175000017500000000226615161702250020576 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/codec/Data.h" #include "eckit/codec/Metadata.h" #include "eckit/exception/Exceptions.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- inline size_t encode_metadata(const std::string& value, Metadata& out) { out.set("type", "string"); out.set("value", value); return 0; } inline void encode_data(const std::string&, Data&) {} inline void decode(const Metadata& metadata, const Data&, std::string& value) { ASSERT(metadata.getString("type") == "string"); metadata.get("value", value); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/types/scalar.h0000664000175000017500000000401415161702250020526 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/codec/Data.h" #include "eckit/codec/Metadata.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const int&, Metadata&); size_t encode_metadata(const long&, Metadata&); size_t encode_metadata(const long long&, Metadata&); size_t encode_metadata(const unsigned long&, Metadata&); size_t encode_metadata(const unsigned long long&, Metadata&); size_t encode_metadata(const float&, Metadata&); size_t encode_metadata(const double&, Metadata&); //--------------------------------------------------------------------------------------------------------------------- void encode_data(const int&, Data&); void encode_data(const long&, Data&); void encode_data(const long long&, Data&); void encode_data(const unsigned long&, Data&); void encode_data(const unsigned long long&, Data&); void encode_data(const float&, Data&); void encode_data(const double&, Data&); //--------------------------------------------------------------------------------------------------------------------- void decode(const Metadata&, const Data&, int&); void decode(const Metadata&, const Data&, long&); void decode(const Metadata&, const Data&, long long&); void decode(const Metadata&, const Data&, unsigned long&); void decode(const Metadata&, const Data&, unsigned long long&); void decode(const Metadata&, const Data&, float&); void decode(const Metadata&, const Data&, double&); //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/types/array/0000775000175000017500000000000015161702250020227 5ustar alastairalastaireckit-2.0.7/src/eckit/codec/types/array/ArrayMetadata.h0000664000175000017500000000750315161702250023124 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/Metadata.h" #include "eckit/codec/detail/DataType.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class ArrayShape : public std::vector { private: using Base = std::vector; public: ArrayShape() = default; ArrayShape(Base&& base) : Base(std::forward(base)) {} template ArrayShape(std::initializer_list list) : Base(list.begin(), list.end()) {} template ArrayShape(idx_t data[], size_t size) : Base(data, data + size) {} template ArrayShape(const std::array& list) : Base(list.begin(), list.end()) {} template ArrayShape(const std::vector& list) : Base(list.begin(), list.end()) {} template >> ArrayShape(Int1 i) { resize(1); operator[](0) = i; } template && std::is_integral_v>> ArrayShape(Int1 i, Int2 j) { resize(2); operator[](0) = i; operator[](1) = j; } template < typename Int1, typename Int2, typename Int3, typename = std::enable_if_t && std::is_integral_v && std::is_integral_v>> ArrayShape(Int1 i, Int2 j, Int3 k) { resize(3); operator[](0) = i; operator[](1) = j; operator[](2) = k; } template && std::is_integral_v && std::is_integral_v && std::is_integral_v>> ArrayShape(Int1 i, Int2 j, Int3 k, Int4 l) { resize(4); operator[](0) = i; operator[](1) = j; operator[](2) = k; operator[](3) = l; } }; //--------------------------------------------------------------------------------------------------------------------- class ArrayMetadata { public: using ArrayShape = eckit::codec::ArrayShape; using DataType = eckit::codec::DataType; static std::string type() { return "array"; } public: ArrayMetadata(); explicit ArrayMetadata(const Metadata&); explicit ArrayMetadata(const DataType&, const ArrayShape&); ArrayMetadata(const ArrayMetadata&); ArrayMetadata(ArrayMetadata&&); ArrayMetadata& operator=(ArrayMetadata&&); int rank() const { return static_cast(shape_.size()); } int shape(int i) const; const ArrayShape& shape() const { return shape_; } DataType datatype() const { return datatype_; } size_t size() const; size_t bytes() const { return size() * datatype_.size(); } friend size_t encode_metadata(const ArrayMetadata& value, Metadata& out); private: ArrayShape shape_; DataType datatype_; }; //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const ArrayMetadata& value, Metadata& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/types/array/ArrayReference.cc0000664000175000017500000000620215161702250023433 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/types/array/ArrayReference.h" #include "eckit/codec/Exceptions.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- void encode_data(const ArrayReference& value, Data& out) { out = Data(value.data(), value.bytes()); } //--------------------------------------------------------------------------------------------------------------------- namespace { template void encode_metadata_value(const ArrayReference& value, Metadata& out) { ASSERT(value.datatype() == make_datatype()); const T* array = reinterpret_cast(value.data()); std::vector vector(value.size()); std::copy(array, array + vector.size(), vector.begin()); out.set("value", vector); } } // namespace size_t encode_metadata(const ArrayReference& value, Metadata& out) { auto bytes = encode_metadata(static_cast(value), out); if (value.rank() == 1 && value.size() <= 4) { auto kind = value.datatype().kind(); using DataType = ArrayReference::DataType; if (kind == DataType::kind()) { encode_metadata_value(value, out); } else if (kind == DataType::kind()) { encode_metadata_value(value, out); } else if (kind == DataType::kind()) { encode_metadata_value(value, out); } else if (kind == DataType::kind()) { encode_metadata_value(value, out); } else if (kind == DataType::kind()) { encode_metadata_value(value, out); } } return bytes; } //--------------------------------------------------------------------------------------------------------------------- ArrayReference::ArrayReference(const void* data, ArrayMetadata::DataType datatype, const ArrayMetadata::ArrayShape& shape) : ArrayMetadata(datatype, shape), data_(const_cast(data)) {} //--------------------------------------------------------------------------------------------------------------------- ArrayReference::ArrayReference(ArrayReference&& other) : ArrayMetadata(std::move(other)), data_(other.data_) { other.data_ = nullptr; } //--------------------------------------------------------------------------------------------------------------------- ArrayReference& ArrayReference::operator=(ArrayReference&& rhs) { ArrayMetadata::operator=(std::move(rhs)); data_ = rhs.data_; rhs.data_ = nullptr; return *this; } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/types/array/adaptors/0000775000175000017500000000000015161702250022044 5ustar alastairalastaireckit-2.0.7/src/eckit/codec/types/array/adaptors/StdArrayAdaptor.h0000664000175000017500000000417715161702250025272 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/codec/Data.h" #include "eckit/codec/Exceptions.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/types/array/ArrayMetadata.h" #include "eckit/codec/types/array/ArrayReference.h" namespace std { //--------------------------------------------------------------------------------------------------------------------- template void interprete(const std::array& vector, eckit::codec::ArrayReference& out) { out = eckit::codec::ArrayReference{vector.data(), {N}}; } //--------------------------------------------------------------------------------------------------------------------- template void decode(const eckit::codec::Metadata& m, const eckit::codec::Data& encoded, std::array& out) { eckit::codec::ArrayMetadata array(m); if (array.datatype().kind() != eckit::codec::ArrayMetadata::DataType::kind()) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector<" << typeid(typename std::decay::type).name() << ">. " << "Incompatible datatype!"; throw eckit::codec::Exception(err.str(), Here()); } if (array.size() != N) { std::stringstream err; err << "Could not decode " << m.json() << " into std::array<" << typeid(typename std::decay::type).name() << "," << N << ">. " << "Incompatible size!"; throw eckit::codec::Exception(err.str(), Here()); } const T* data = static_cast(encoded.data()); std::copy(data, data + N, out.begin()); } //--------------------------------------------------------------------------------------------------------------------- } // end namespace std eckit-2.0.7/src/eckit/codec/types/array/adaptors/StdVectorAdaptor.h0000664000175000017500000000407215161702250025450 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/codec/Data.h" #include "eckit/codec/Exceptions.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/detail/TypeTraits.h" #include "eckit/codec/types/array/ArrayMetadata.h" #include "eckit/codec/types/array/ArrayReference.h" namespace std { //--------------------------------------------------------------------------------------------------------------------- template = 0> void interprete(const std::vector& vector, eckit::codec::ArrayReference& out) { using eckit::codec::ArrayReference; out = ArrayReference{vector.data(), {int(vector.size())}}; } //--------------------------------------------------------------------------------------------------------------------- template = 0> void decode(const eckit::codec::Metadata& m, const eckit::codec::Data& encoded, std::vector& out) { eckit::codec::ArrayMetadata array(m); if (array.datatype().kind() != eckit::codec::ArrayMetadata::DataType::kind()) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector<" << eckit::codec::demangle() << ">. " << "Incompatible datatypes: " << array.datatype().str() << " and " << eckit::codec::ArrayMetadata::DataType::str() << "."; throw eckit::codec::Exception(err.str(), Here()); } const auto* data = static_cast(encoded.data()); out.assign(data, data + array.size()); } //--------------------------------------------------------------------------------------------------------------------- } // end namespace std eckit-2.0.7/src/eckit/codec/types/array/adaptors/StdVectorOfStdArrayAdaptor.h0000664000175000017500000000516315161702250027411 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/codec/Data.h" #include "eckit/codec/Exceptions.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/detail/demangle.h" #include "eckit/codec/types/array/ArrayMetadata.h" #include "eckit/codec/types/array/ArrayReference.h" namespace std { //--------------------------------------------------------------------------------------------------------------------- template void interprete(const std::vector>& vector_of_array, eckit::codec::ArrayReference& out) { using eckit::codec::ArrayReference; out = ArrayReference{vector_of_array.front().data(), {vector_of_array.size(), N}}; } //--------------------------------------------------------------------------------------------------------------------- template void decode(const eckit::codec::Metadata& m, const eckit::codec::Data& encoded, std::vector>& out) { eckit::codec::ArrayMetadata array(m); if (array.datatype().kind() != eckit::codec::ArrayMetadata::DataType::kind()) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector<" << eckit::codec::demangle() << ">. " << "Incompatible datatype!"; throw eckit::codec::Exception(err.str(), Here()); } if (array.rank() != 2) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector() << "," << N << ">>. " << "Incompatible rank!"; throw eckit::codec::Exception(err.str(), Here()); } if (array.shape(1) != N) { std::stringstream err; err << "Could not decode " << m.json() << " into std::vector() << "," << N << ">>. " << "Incompatible size!"; throw eckit::codec::Exception(err.str(), Here()); } const auto* data = static_cast*>(encoded.data()); // std::copy(data, data + array.shape(0), out.begin()); out.assign(data, data + array.shape(0)); } //--------------------------------------------------------------------------------------------------------------------- } // end namespace std eckit-2.0.7/src/eckit/codec/types/array/ArrayReference.h0000664000175000017500000000315415161702250023300 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/codec/Data.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/types/array/ArrayMetadata.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class ArrayReference : public ArrayMetadata { public: ArrayReference() = default; ArrayReference(const void* data, DataType, const ArrayShape&); template ArrayReference(const T* data, const ArrayShape& shape) : ArrayMetadata(DataType::create(), shape), data_(const_cast(data)) {} ArrayReference(ArrayReference&&); ArrayReference& operator=(ArrayReference&&); void* data() const { return data_; } friend void decode(const Metadata&, const Data&, ArrayReference&); private: void* data_{nullptr}; }; //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const ArrayReference& value, Metadata& out); void encode_data(const ArrayReference& value, Data& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/types/array/ArrayMetadata.cc0000664000175000017500000000713315161702250023261 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/types/array/ArrayMetadata.h" #include #include #include #include "eckit/codec/Exceptions.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const ArrayMetadata& value, Metadata& out) { out.set("type", ArrayMetadata::type()); out.set("shape", value.shape_); out.set("datatype", value.datatype_.str()); return value.bytes(); } //--------------------------------------------------------------------------------------------------------------------- int ArrayMetadata::shape(int i) const { if (i >= rank()) { throw Exception( "ArrayMetadata::shape(i=" + std::to_string(i) + ") goes out of bounds. rank=" + std::to_string(rank()), Here()); } return static_cast(shape_[static_cast(i)]); } //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata(const Metadata& metadata) : datatype_(DataType::KIND_REAL64) /* circumvent absense of default constructor */ { std::string encoded_type; ASSERT_MSG(metadata.get("type", encoded_type), "metadata is missing 'type'"); ASSERT_MSG(encoded_type == type(), "metadata has unexpected type '" + encoded_type + "'"); ASSERT_MSG(metadata.get("shape", shape_), "metadata is missing 'shape'"); std::string datatype_str; ASSERT_MSG(metadata.get("datatype", datatype_str), "metadata is missing 'datatype'"); datatype_ = DataType(datatype_str); } //--------------------------------------------------------------------------------------------------------------------- size_t ArrayMetadata::size() const { return size_t(std::accumulate(shape_.begin(), shape_.end(), 1, std::multiplies<>())); } //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata() : datatype_(DataType::KIND_REAL64) /* circumvent absense of default constructor */ {} //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata(const DataType& datatype, const ArrayShape& shape) : shape_(shape), datatype_(datatype) {} //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata(const ArrayMetadata& other) : ArrayMetadata{other.datatype_, other.shape_} {} //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata::ArrayMetadata(ArrayMetadata&& other) : shape_(std::move(other.shape_)), datatype_{other.datatype_} {} //--------------------------------------------------------------------------------------------------------------------- ArrayMetadata& ArrayMetadata::operator=(ArrayMetadata&& rhs) { shape_ = std::move(rhs.shape_); datatype_ = rhs.datatype_; return *this; } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/RecordWriter.h0000664000175000017500000001056115161702250020534 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/FileStream.h" #include "eckit/codec/RecordItem.h" #include "eckit/codec/Stream.h" #include "eckit/codec/detail/Defaults.h" #include "eckit/codec/detail/Encoder.h" #include "eckit/codec/detail/NoConfig.h" #include "eckit/codec/detail/Reference.h" #include "eckit/codec/detail/TypeTraits.h" #include "eckit/codec/types/array/ArrayReference.h" #include "eckit/codec/types/scalar.h" #include "eckit/codec/types/string.h" namespace eckit::codec { template Interpreted interprete(T& in) { Interpreted interpreted; interprete(in, interpreted); return interpreted; } //--------------------------------------------------------------------------------------------------------------------- /// @class RecordWriter /// @brief Write record class RecordWriter { public: using Key = std::string; public: /// @brief Set compression void compression(const std::string&); /// @brief Set compression off or to default void compression(bool); /// @brief Set checksum off or to default void checksum(bool); // -- set( Key, Value ) where Value can be a variety of things /// @brief Add link to other record item (RecordItem::URI) void set(const Key&, Link&&, const Configuration& = NoConfig()); /// @brief Add item to record void set(const Key&, Encoder&&, const Configuration& = NoConfig()); /// @brief Add item to record template = 0> void set(const Key& key, Value&& value, const Configuration& config = NoConfig()) { set(key, Encoder{std::move(value)}, config); } /// @brief Add item to record template void set(const Key& key, const Reference& value, const Configuration& config = NoConfig()) { set(key, std::move(value), config); } /// @brief Add item to record template = 0> void set(const Key& key, const Value& value, const Configuration& config = NoConfig()) { set(key, RecordItem(interprete(value)), config); } /// @brief Add item to record template = 0> void set(const Key& key, const Value& value, const Configuration& config = NoConfig()) { set(key, Encoder{value}, config); } void set(const Key& key, const char* value, const Configuration& config = NoConfig()) { set(key, Encoder{std::string(value)}, config); } void set(const Key& key, const std::string& value, const Configuration& config = NoConfig()) { set(key, Encoder{std::string(value)}, config); } /// @brief Write new record to path size_t write(const PathName&, Mode = Mode::write) const; /// @brief Write new record to a DataHandle /// @pre The DataHandle must be opened for Write access. size_t write(DataHandle&) const; /// @brief Write new record to a Stream /// @pre The Stream must be opened for Write access. size_t write(Stream) const; /// @brief estimate maximum size of record /// /// This could be useful to write a record to a fixed size MemoryHandle /// /// @note Without compression this matches exactly the required record size. /// With compression active, the data sizes are assumed to be 120% of uncompressed sizes for robustness, /// which may seem contradictory. size_t estimateMaximumSize() const; private: std::vector keys_; std::map encoders_; std::map info_; std::string compression_{defaults::compression_algorithm()}; int do_checksum_{defaults::checksum_write() ? 1 : 0}; int nb_data_sections_{0}; std::string metadata() const; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Record.cc0000664000175000017500000002604015161702250017474 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/Record.h" #include "eckit/codec/Exceptions.h" #include "eckit/codec/detail/ParsedRecord.h" #include "eckit/codec/detail/Version.h" #include "eckit/config/YAMLConfiguration.h" #include "eckit/filesystem/URI.h" namespace eckit::codec { namespace { //--------------------------------------------------------------------------------------------------------------------- template inline size_t read_struct(IStream& in, Struct& s) { static_assert(Struct::bytes == sizeof(Struct)); return in.read(reinterpret_cast(&s), sizeof(Struct)); } //--------------------------------------------------------------------------------------------------------------------- template inline Struct read_struct(IStream& in) { Struct s; if (read_struct(in, s) != sizeof(Struct)) { throw InvalidRecord("Unexpected EOF reached"); } return s; } //--------------------------------------------------------------------------------------------------------------------- } // namespace //--------------------------------------------------------------------------------------------------------------------- Endian RecordHead::endian() const { return magic_number == 1234 ? Endian::native : magic_number == 3523477504 ? Endian::swapped : throw Exception("Mixed endianness is not supported", Here()); } //--------------------------------------------------------------------------------------------------------------------- std::string Record::URI::str() const { eckit::URI uri("file", PathName(path)); uri.query("offset", std::to_string(offset)); return uri.asRawString(); } //--------------------------------------------------------------------------------------------------------------------- Record::Record() : record_(new ParsedRecord()) {} //--------------------------------------------------------------------------------------------------------------------- bool Record::empty() const { return record_->head.record_length == 0; } //--------------------------------------------------------------------------------------------------------------------- const Metadata& Record::metadata(const std::string& key) const { if (record_->items.find(key) == record_->items.end()) { throw Exception("Record does not contain key \"" + key + "\"", Here()); } return record_->items.at(key); } //--------------------------------------------------------------------------------------------------------------------- Endian Record::endian() const { return record_->head.endian(); } //--------------------------------------------------------------------------------------------------------------------- Version Record::version() const { return record_->head.version; } //--------------------------------------------------------------------------------------------------------------------- Time Record::time() const { return record_->head.time; } //--------------------------------------------------------------------------------------------------------------------- uint64_t Record::size() const { return record_->head.record_length; } //--------------------------------------------------------------------------------------------------------------------- const std::vector& Record::keys() const { return record_->keys; } //--------------------------------------------------------------------------------------------------------------------- bool Record::has(const std::string& key) { return record_->items.find(key) != record_->items.end(); } //--------------------------------------------------------------------------------------------------------------------- Record::operator const ParsedRecord&() const { return *record_; } //--------------------------------------------------------------------------------------------------------------------- static void parse_record(ParsedRecord& record, const std::string& key, const Metadata& metadata) { if (metadata.type() || metadata.link()) { record.items.emplace(key, metadata); record.keys.emplace_back(key); } else { for (auto& next_key : metadata.keys()) { parse_record(record, key + "." + next_key, metadata.getSubConfiguration(next_key)); } } } //--------------------------------------------------------------------------------------------------------------------- Record& Record::read(Stream& in, bool read_to_end) { if (not empty()) { return *this; } auto& r = record_->head; auto rbegin = in.position(); // Begin Record // ------------ if (read_struct(in, r) != sizeof(r)) { if (in.position() > sizeof(r.begin.string)) { if (not r.begin.valid()) { std::stringstream err; err << "Format is not recognized. Received: " << r.begin.string; throw InvalidRecord(err.str()); } } throw InvalidRecord("Unexpected EOF reached"); } if (not r.valid()) { std::stringstream err; err << "Format is not recognized. Received: " << r.begin.string; throw InvalidRecord(err.str()); } if (r.version < RecordHead{}.version) { throw InvalidRecord("Version of record (" + r.version.str() + ") is too old."); } if (r.metadata_length < sizeof(RecordMetadataSection::Begin) + sizeof(RecordMetadataSection::End)) { throw InvalidRecord("Unexpected metadata section length: " + std::to_string(r.metadata_length) + " < " + std::to_string(sizeof(RecordMetadataSection::Begin) + sizeof(RecordMetadataSection::End))); } if (r.index_length < sizeof(RecordDataIndexSection::Begin) + sizeof(RecordDataIndexSection::End)) { throw InvalidRecord("Unexpected data index section length."); } r.metadata_offset += rbegin; r.index_offset += rbegin; // Metadata section // ---------------- in.seek(r.metadata_offset); auto metadata_begin = read_struct(in); if (not metadata_begin.valid()) { throw InvalidRecord("Metadata section is not valid. Invalid section begin marker: [" + metadata_begin.str() + "]"); } std::string metadata_str; metadata_str.resize(static_cast(r.metadata_length) - sizeof(RecordMetadataSection::Begin) - sizeof(RecordMetadataSection::End)); if (in.read(const_cast(metadata_str.data()), metadata_str.size()) != metadata_str.size()) { throw InvalidRecord("Unexpected EOF reached"); } auto metadata_end = read_struct(in); if (not metadata_end.valid()) { throw InvalidRecord("Metadata section is not valid. Invalid section end marker: [" + metadata_end.str() + "]"); } Checksum encoded_metadata_checksum(r.metadata_checksum); Checksum computed_metadata_checksum( checksum(metadata_str.data(), metadata_str.size(), encoded_metadata_checksum.algorithm())); if (computed_metadata_checksum.available() && encoded_metadata_checksum.str() != computed_metadata_checksum.str()) { std::stringstream err; err << "Mismatch in metadata checksum.\n"; err << " Encoded: [" << encoded_metadata_checksum.str() << "].\n"; err << " Computed: [" << computed_metadata_checksum.str() << "]."; throw DataCorruption(err.str()); } ASSERT(r.metadata_format == "yaml"); Metadata metadata = YAMLConfiguration(metadata_str); for (auto& key : metadata.keys()) { parse_record(*record_, key, metadata.getSubConfiguration(key)); } // DataIndex section // ----------------- in.seek(r.index_offset); auto index_begin = read_struct(in); if (not index_begin.valid()) { throw InvalidRecord("Data index section is not valid. Invalid section begin marker: [" + index_begin.str() + "]"); } const auto index_length = (static_cast(r.index_length) - sizeof(RecordDataIndexSection::Begin) - sizeof(RecordDataIndexSection::End)); const auto index_size = index_length / sizeof(RecordDataIndexSection::Entry); auto& data_sections = record_->data_sections; data_sections.resize(index_size); if (in.read(data_sections.data(), index_length) != index_length) { throw InvalidRecord("Unexpected EOF reached"); } auto index_end = read_struct(in); if (not index_end.valid()) { throw InvalidRecord("Data index section is not valid. Invalid section end marker: [" + index_end.str() + "]"); } for (auto& data_section : data_sections) { data_section.offset += rbegin; if (data_section.length < sizeof(RecordDataSection::Begin) + sizeof(RecordDataSection::End)) { throw InvalidRecord("Unexpected data section length: [" + std::to_string(data_section.length) + "]"); } } record_->parse(); if (read_to_end) { in.seek(rbegin + r.record_length - sizeof(RecordEnd)); RecordEnd record_end; read_struct(in, record_end); if (not record_end.valid()) { throw InvalidRecord("RecordEnd has unexpected format: [" + record_end.str() + "]"); } if (in.position() != rbegin + r.record_length) { throw InvalidRecord("RecordEnd has unexpected format"); } } return *this; } //--------------------------------------------------------------------------------------------------------------------- void ParsedRecord::parse() { for (auto& key : keys) { auto& item = items.at(key); item.record.version_ = head.version; item.record.created_ = head.time; item.data.section(item.getInt("data.section", 0)); item.data.endian(head.endian()); item.data.compression(item.getString("data.compression.type", "none")); if (item.data.section() != 0) { auto& data_section = data_sections.at(static_cast(item.data.section() - 1)); item.data.checksum(data_section.checksum); item.data.compressed_size(data_section.length - sizeof(RecordDataSection::Begin) - sizeof(RecordDataSection::End)); if (item.data.compressed()) { item.data.size(uncompressed_size(item)); } else { item.data.size(item.data.compressed_size()); } } } } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/RecordPrinter.h0000664000175000017500000000331515161702250020702 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/Record.h" #include "eckit/codec/Session.h" #include "eckit/codec/detail/NoConfig.h" #include "eckit/config/Configuration.h" #include "eckit/filesystem/PathName.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class RecordPrinter { public: explicit RecordPrinter(const Record::URI&, const Configuration& = NoConfig()); explicit RecordPrinter(const PathName&, const Configuration& = NoConfig()); RecordPrinter(const PathName&, std::uint64_t offset, const Configuration& = NoConfig()); Record record() const { return record_; } size_t size() const { return record_.size(); } Version version() const { return record_.version(); } Time time() const { return record_.time(); } void print(std::ostream& out) const; friend std::ostream& operator<<(std::ostream&, const RecordPrinter&); private: Session session_; Record::URI uri_; struct { std::string format{"table"}; bool details{false}; } options_; Record record_; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Stream.h0000664000175000017500000000520615161702250017354 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit { class DataHandle; } namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- /// @class Stream /// @brief Handle to a shared eckit::DataHandle /// /// Note, a Stream is not intended to be opened and closed within eckit::codec context /// The derived classes InputFileStream and OutputFileStream automatically open and close /// on construction an destruction. class Stream { public: /// Default constructor /// @post Stream is not usable but can be assigned to become valid Stream() = default; /// Constructor taking ownership of datahandle explicit Stream(DataHandle*); /// Constructor to share datahandle with a shared_ptr explicit Stream(std::shared_ptr); /// Constructor referencing datahandle, no ownership is taken /// @note The usability depends on the usable lifetime of /// the referenced datahandle Stream(DataHandle&); /// Assignment/Copy constructor sharing datahandle with other Stream Stream(const Stream&) = default; Stream(Stream&&) = default; Stream& operator=(const Stream&) = default; Stream& operator=(Stream&&) = default; /// Access internal DataHandle DataHandle& datahandle(); /// Move position to given offset std::uint64_t seek(std::uint64_t offset); /// Return offset of current position std::uint64_t position(); /// Write data of given length (bytes) /// @return number of bytes written /// @post The position is increased with number of bytes written std::uint64_t write(const void* data, size_t length); /// Read data of given length (bytes) /// @return number of bytes read /// @post The position is increased with number of bytes read std::uint64_t read(void* data, size_t length); /// Return true if pointer is valid; explicit operator bool() const { return ptr_ != nullptr; } private: std::shared_ptr shared_; DataHandle* ptr_{nullptr}; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Metadata.cc0000664000175000017500000000441415161702250017777 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/Metadata.h" #include #include #include #include "eckit/codec/Exceptions.h" #include "eckit/codec/types/array/ArrayReference.h" #include "eckit/log/JSON.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- size_t uncompressed_size(const Metadata& m) { if (m.has("data.size")) { return m.getUnsigned("data.size"); } if (m.has("type") && m.getString("type") == "array") { ArrayMetadata array(m); return array.bytes(); } std::stringstream err; err << "Could not compute uncompressed data size from metadata \n"; write(m, err); throw Exception(err.str()); } //--------------------------------------------------------------------------------------------------------------------- void write(const Metadata& metadata, std::ostream& out) { JSON js(out, JSON::Formatting::indent(4)); js << metadata; } void write(const Metadata& metadata, Stream& out) { std::stringstream ss; write(metadata, ss); std::string s = ss.str(); out.write(s.data(), s.size()); } //--------------------------------------------------------------------------------------------------------------------- void Metadata::link(Metadata&& linked) { std::string initial_link = link(); ASSERT(!initial_link.empty()); data = std::move(linked.data); record = std::move(linked.record); set(linked); // Set link to initial_link, in case the link is itself another link set("link", initial_link); } std::string Metadata::json() const { std::stringstream s; JSON js(s, JSON::Formatting::compact()); js << *this; return s.str(); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/0000775000175000017500000000000015161702250017207 5ustar alastairalastaireckit-2.0.7/src/eckit/codec/detail/Encoder.cc0000664000175000017500000000143115161702250021074 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/detail/Encoder.h" #include "eckit/codec/Exceptions.h" namespace eckit::codec { size_t encode_metadata(const Encoder& encoder, Metadata& metadata) { ASSERT(encoder); return encoder.self_->encode_metadata_(metadata); } void encode_data(const Encoder& encoder, Data& out) { ASSERT(encoder); encoder.self_->encode_data_(out); } } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/RecordInfo.h0000664000175000017500000000131715161702250021414 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/codec/detail/Time.h" #include "eckit/codec/detail/Version.h" namespace eckit::codec { struct RecordInfo { Version version_; Time created_; const Version& version() const { return version_; } const Time& created() const { return created_; } }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Time.cc0000664000175000017500000000416615161702250020423 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/detail/Time.h" #include #include #include #include // std::ios_base #include // std::ostreambuf_iterator #include // std::use_facet, std::time_put #include // std::basic_ostream #include #include #include "eckit/log/JSON.h" namespace eckit::codec { namespace { std::time_t to_time_t(Time time) { return static_cast(time.tv_sec); } Time from_time_point(std::chrono::time_point t) { auto since_epoch = t.time_since_epoch(); auto sec_since_epoch = std::chrono::duration_cast(since_epoch); auto nsec_since_epoch = std::chrono::duration_cast(since_epoch); auto extra_nsec = std::chrono::duration_cast(nsec_since_epoch - sec_since_epoch); Time time; time.tv_sec = static_cast(sec_since_epoch.count()); time.tv_nsec = static_cast(extra_nsec.count()); return time; } } // namespace Time Time::now() { return from_time_point(std::chrono::system_clock::now()); } void Time::print(std::ostream& out) const { // Will print time-date in ISO 8601 format: 1970-01-01T00:00:00.123456789Z auto time = to_time_t(*this); out << std::put_time(::gmtime(&time), "%FT%T") << "." << tv_nsec << "Z"; } std::ostream& operator<<(std::ostream& out, const Time& time) { time.print(out); return out; } JSON& operator<<(JSON& out, const Time& time) { std::stringstream s; s << time; out << s.str(); return out; } std::string Time::str() const { std::stringstream s; print(s); return s.str(); } } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Time.h0000664000175000017500000000223515161702250020260 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include namespace eckit { class JSON; } namespace eckit::codec { /// Store UTC time up to nanosecond precision struct Time { std::uint64_t tv_sec{0}; ///< seconds since Epoch (1970-01-01T00:00:00Z) std::uint64_t tv_nsec{0}; ///< additional nanoseconds /// Create current time using system clock static Time now(); /// print UTC time in ISO 8601 format: "1970-01-01T00:00:00.123456789Z" void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream&, const Time&); friend JSON& operator<<(JSON&, const Time&); /// @return string of UTC time in ISO 8601 format: "1970-01-01T00:00:00.123456789Z" std::string str() const; }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Link.h0000664000175000017500000000150015161702250020251 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include namespace eckit { class PathName; } namespace eckit::codec { struct Link { std::string uri; const std::string& str() const { return uri; } operator bool() const { return !uri.empty(); } operator const std::string&() const { return str(); } // bool relative() const; // friend Link operator/( const PathName& dir, const Link& link ); }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/tag.h0000664000175000017500000000103615161702250020133 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once namespace eckit::codec::tag { struct disable_static_assert {}; struct enable_static_assert {}; } // namespace eckit::codec::tag eckit-2.0.7/src/eckit/codec/detail/DataType.h0000664000175000017500000002451115161702250021076 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include //------------------------------------------------------------------------------------------------------ namespace eckit::codec { class DataType { public: using kind_t = long; static const kind_t KIND_BYTE = 1; static const kind_t KIND_INT32 = -4; static const kind_t KIND_INT64 = -8; static const kind_t KIND_REAL32 = 4; static const kind_t KIND_REAL64 = 8; static const kind_t KIND_UINT64 = -16; template static DataType create(); static DataType byte() { return {KIND_BYTE}; } static DataType int32() { return {KIND_INT32}; } static DataType int64() { return {KIND_INT64}; } static DataType real32() { return {KIND_REAL32}; } static DataType real64() { return {KIND_REAL64}; } static DataType uint64() { return {KIND_UINT64}; } template static kind_t kind(); template static kind_t kind(const DATATYPE&); template static std::string str(); template static std::string str(const DATATYPE); static kind_t str_to_kind(const std::string&); static std::string kind_to_str(kind_t); static bool kind_valid(kind_t); private: static std::string byte_str() { return "byte"; } static std::string int32_str() { return "int32"; } static std::string int64_str() { return "int64"; } static std::string real32_str() { return "real32"; } static std::string real64_str() { return "real64"; } static std::string uint64_str() { return "uint64"; } [[noreturn]] static void throw_not_recognised(kind_t); [[noreturn]] static void throw_not_recognised(std::string datatype); public: explicit DataType(const std::string&); DataType(long); DataType(const DataType&) = default; DataType& operator=(const DataType&) = default; std::string str() const { return kind_to_str(kind_); } kind_t kind() const { return kind_; } size_t size() const { return (kind_ == KIND_UINT64) ? 8 : std::abs(kind_); } friend bool operator==(DataType dt1, DataType dt2); friend bool operator!=(DataType dt1, DataType dt2); friend bool operator==(DataType dt, kind_t kind); friend bool operator!=(DataType dt, kind_t kind); friend bool operator==(kind_t kind, DataType dt); friend bool operator!=(kind_t kind, DataType dt2); private: kind_t kind_; }; template <> inline std::string DataType::str() { return byte_str(); } template <> inline std::string DataType::str() { return byte_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(int) == 4); return int32_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(int) == 4); return int32_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(long) == 8); return int64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(long) == 8); return int64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(long long) == 8); return int64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(long long) == 8); return int64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(float) == 4); return real32_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(float) == 4); return real32_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(double) == 8); return real64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(double) == 8); return real64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(unsigned long) == 8); return uint64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(unsigned long) == 8); return uint64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(unsigned long long) == 8); return uint64_str(); } template <> inline std::string DataType::str() { static_assert(sizeof(unsigned long long) == 8); return uint64_str(); } template <> inline std::string DataType::str(const int&) { return str(); } template <> inline std::string DataType::str(const long&) { return str(); } template <> inline std::string DataType::str(const long long&) { return str(); } template <> inline std::string DataType::str(const unsigned long&) { return str(); } template <> inline std::string DataType::str(const unsigned long long&) { return str(); } template <> inline std::string DataType::str(const float&) { return str(); } template <> inline std::string DataType::str(const double&) { return str(); } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(std::byte) == 1); return KIND_BYTE; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(std::byte) == 1); return KIND_BYTE; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(int) == 4); return KIND_INT32; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(int) == 4); return KIND_INT32; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(long) == 8); return KIND_INT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(long) == 8); return KIND_INT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(long long) == 8); return KIND_INT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(long long) == 8); return KIND_INT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(unsigned long) == 8); return KIND_UINT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(unsigned long) == 8); return KIND_UINT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(unsigned long long) == 8); return KIND_UINT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(unsigned long long) == 8); return KIND_UINT64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(float) == 4); return KIND_REAL32; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(float) == 4); return KIND_REAL32; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(double) == 8); return KIND_REAL64; } template <> inline DataType::kind_t DataType::kind() { static_assert(sizeof(double) == 8); return KIND_REAL64; } template <> inline DataType::kind_t DataType::kind(const int&) { return kind(); } template <> inline DataType::kind_t DataType::kind(const long&) { return kind(); } template <> inline DataType::kind_t DataType::kind(const unsigned long&) { return kind(); } template <> inline DataType::kind_t DataType::kind(const float&) { return kind(); } template <> inline DataType::kind_t DataType::kind(const double&) { return kind(); } inline DataType::kind_t DataType::str_to_kind(const std::string& datatype) { if (datatype == "int32") { return KIND_INT32; } if (datatype == "int64") { return KIND_INT64; } if (datatype == "uint64") { return KIND_UINT64; } if (datatype == "real32") { return KIND_REAL32; } if (datatype == "real64") { return KIND_REAL64; } if (datatype == "byte") { return KIND_BYTE; } throw_not_recognised(datatype); } inline std::string DataType::kind_to_str(kind_t kind) { switch (kind) { case KIND_INT32: return int32_str(); case KIND_INT64: return int64_str(); case KIND_UINT64: return uint64_str(); case KIND_REAL32: return real32_str(); case KIND_REAL64: return real64_str(); case KIND_BYTE: return byte_str(); default: throw_not_recognised(kind); } } inline bool DataType::kind_valid(kind_t kind) { switch (kind) { case KIND_BYTE: case KIND_INT32: case KIND_INT64: case KIND_UINT64: case KIND_REAL32: case KIND_REAL64: return true; default: return false; } } inline DataType::DataType(const std::string& datatype) : kind_(str_to_kind(datatype)) {} inline DataType::DataType(long kind) : kind_(kind) {} inline bool operator==(DataType dt1, DataType dt2) { return dt1.kind_ == dt2.kind_; } inline bool operator!=(DataType dt1, DataType dt2) { return dt1.kind_ != dt2.kind_; } inline bool operator==(DataType dt, DataType::kind_t kind) { return dt.kind_ == kind; } inline bool operator!=(DataType dt, DataType::kind_t kind) { return dt.kind_ != kind; } inline bool operator==(DataType::kind_t kind, DataType dt) { return dt.kind_ == kind; } inline bool operator!=(DataType::kind_t kind, DataType dt) { return dt.kind_ != kind; } template inline DataType DataType::create() { return DataType(DataType::kind()); } template inline DataType make_datatype() { return DataType(DataType::kind()); } //------------------------------------------------------------------------------------------------------ } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Decoder.cc0000664000175000017500000000134515161702250021066 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/detail/Decoder.h" namespace eckit::codec { void decode(const Metadata& metadata, const Data& data, Decoder& decoder) { decoder.self_->decode_(metadata, data); } void decode(const Metadata& metadata, const Data& data, Decoder&& decoder) { decoder.self_->decode_(metadata, data); } } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Checksum.h0000664000175000017500000000163415161702250021126 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit::codec { class Checksum { public: Checksum() = default; explicit Checksum(const std::string& checksum); bool available() const; std::string str() const; std::string str(size_t size) const; std::string algorithm() const { return algorithm_; } private: std::string algorithm_; std::string checksum_; }; std::string checksum(const void* buffer, size_t size, const std::string& algorithm = ""); } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Base64.h0000664000175000017500000000230515161702250020404 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class Base64 { public: static std::string encode(const void* data, size_t len); static std::string decode(const void* data, size_t len); template static std::string encode(const T& value) { return encode(&value, sizeof(value)); } template static T decode(const std::string& in) { std::string decoded = decode(in.data(), in.size()); return *reinterpret_cast(decoded.data()); } }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Defaults.h0000664000175000017500000000246715161702250021140 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/config/Resource.h" namespace eckit::codec::defaults { [[maybe_unused]] static const std::string& checksum_algorithm() { static const auto checksum = Resource("eckit.codec.checksum.algorithm;$ECKIT_CODEC_CHECKSUM", "xxh64"); return checksum; } [[maybe_unused]] static bool checksum_read() { static const auto checksum = Resource("eckit.codec.checksum.read;$ECKIT_CODEC_CHECKSUM_READ", true); return checksum; } [[maybe_unused]] static bool checksum_write() { static const auto checksum = Resource("eckit.codec.checksum.write;$ECKIT_CODEC_CHECKSUM_WRITE", true); return checksum; } [[maybe_unused]] static const std::string& compression_algorithm() { static const auto compression = Resource("eckit.codec.compression;$ECKIT_CODEC_COMPRESSION", "none"); return compression; } } // namespace eckit::codec::defaults eckit-2.0.7/src/eckit/codec/detail/Reference.h0000664000175000017500000000217215161702250021260 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/codec/Data.h" #include "eckit/codec/Exceptions.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/detail/sfinae.h" namespace eckit::codec { template struct Reference { const T* ref; explicit Reference(const T& r) : ref(&r) {} friend size_t encode_metadata(const Reference& in, Metadata& metadata) { size_t size{0}; if (not sfinae::encode_metadata(*in.ref, metadata, size)) { throw NotEncodable(*in.ref); } return size; } friend void encode_data(const Reference& in, Data& out) { if (not sfinae::encode_data(*in.ref, out)) { throw NotEncodable(*in.ref); } } }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/RecordSections.h0000664000175000017500000001407115161702250022311 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/codec/detail/Endian.h" #include "eckit/codec/detail/Time.h" #include "eckit/codec/detail/Version.h" #include "eckit/types/FixedString.h" namespace eckit::codec { // ------------------------------------------------------------------------------------------------------------------------------------ constexpr char RECORD_BEGIN[] = "ATLAS-IO"; constexpr char RECORD_END[] = "ATLAS-IO-END"; struct RecordBegin { FixedString<8> string{RECORD_BEGIN}; FixedString<8> spare{"\n"}; bool valid() const { return string == RECORD_BEGIN; } std::string str() const { return string; } }; struct RecordEnd { // 32 bytes static constexpr size_t bytes = 32; FixedString<1> eol{"\n"}; FixedString<12> string{RECORD_END}; FixedString<19> padding{" \n\n\n\n"}; bool valid() const { return string == RECORD_END; } std::string str() const { return string; } }; // ------------------------------------------------------------------------------------------------------------------------------------ struct RecordHead { static constexpr size_t bytes = 256; static constexpr size_t padding_ = bytes - 16 - 8 - 16 - 8 * 4 - 64 - 8 * 2 - 4 - 1; RecordBegin begin; ///< 16 beginning of record Version version; ///< 8 version of this record Time time; ///< 16 time since system_clock epoch (1970-1-1 00:00) uint64_t record_length{0}; ///< 8 length of entire record FixedString<8> metadata_format{"yaml"}; ///< 8 format of metadata section in this record std::uint64_t metadata_offset{bytes}; ///< 8 offset where metadata section starts std::uint64_t metadata_length{0}; ///< 8 length of metadata section FixedString<64> metadata_checksum; ///< 64 checksum of metadata std::uint64_t index_offset{0}; ///< 8 offset where data section starts std::uint64_t index_length; ///< 8 length of data section std::uint32_t magic_number{1234}; ///< 4 number 1234 encoded in binary, used to detect encoded endianness FixedString padding__; ///< Extra padding to get to FixedString<1> eol{"\n"}; static constexpr size_t size() { return bytes; } ///< Size in bytes of this section Endian endian() const; ///< Endianness determined from magic_number bool valid() const { return begin.valid(); } ///< Check if this is a valid RecordRoot std::string str() const { return begin.str(); } }; // ------------------------------------------------------------------------------------------------------------------------------------ struct RecordMetadataSection { static constexpr char TAG_BEGIN[] = "METADATA-BEGIN"; static constexpr char TAG_END[] = "METADATA-END"; struct Begin { // 32 bytes static constexpr size_t bytes = 32; FixedString<1> eol{"\n"}; FixedString<14> string{TAG_BEGIN}; FixedString<17> padding{" \n"}; bool valid() const { return string == TAG_BEGIN; } std::string str() const { return string; } }; struct End { // 32 bytes static constexpr size_t bytes = 32; FixedString<1> eol{"\n"}; FixedString<12> string{TAG_END}; FixedString<19> padding{" \n"}; bool valid() const { return string == TAG_END; } std::string str() const { return string; } }; }; // ------------------------------------------------------------------------------------------------------------------------------------ struct RecordDataIndexSection { static constexpr char TAG_BEGIN[] = "INDEX-BEGIN"; static constexpr char TAG_END[] = "INDEX-END"; struct Begin { // 32 bytes static constexpr size_t bytes = 32; FixedString<1> eol{"\n"}; FixedString<11> string{TAG_BEGIN}; FixedString<20> padding{" \n"}; bool valid() const { return string == TAG_BEGIN; } std::string str() const { return string; } }; struct End { // 32 bytes static constexpr size_t bytes = 32; FixedString<1> eol{"\n"}; FixedString<9> string{TAG_END}; FixedString<22> padding{" \n"}; bool valid() const { return string == TAG_END; } std::string str() const { return string; } }; struct Entry { static constexpr size_t bytes = 80; std::uint64_t offset; std::uint64_t length; FixedString<64> checksum; }; }; // ------------------------------------------------------------------------------------------------------------------------------------ struct RecordDataSection { static constexpr char TAG_BEGIN[] = "DATA-BEGIN"; static constexpr char TAG_END[] = "DATA-END"; struct Begin { // 32 bytes static constexpr size_t bytes = 32; FixedString<1> eol{"\n"}; FixedString<10> string{TAG_BEGIN}; FixedString<21> padding{" \n"}; bool valid() const { return string == TAG_BEGIN; } std::string str() const { return string; } }; struct End { // 32 bytes static constexpr size_t bytes = 32; FixedString<1> eol{"\n"}; FixedString<8> string{TAG_END}; FixedString<23> padding{" \n"}; bool valid() const { return string == TAG_END; } std::string str() const { return string; } }; }; // ------------------------------------------------------------------------------------------------------------------------------------ } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/demangle.cc0000664000175000017500000000217615161702250021300 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/detail/demangle.h" #include "eckit/eckit_config.h" #if eckit_HAVE_CXXABI_H #include #include #endif namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- std::string demangle(const char* mangled_name) { #if eckit_HAVE_CXXABI_H int status = -4; std::unique_ptr res{abi::__cxa_demangle(mangled_name, nullptr, nullptr, &status), std::free}; return status == 0 ? res.get() : mangled_name; #else return {mangled_name}; #endif } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/TypeTraits.h0000664000175000017500000001154115161702250021472 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/codec/detail/DataType.h" namespace eckit::codec { class Metadata; class Data; namespace adl_tests { using std::declval; using std::is_pod; template std::false_type can_interprete(...) noexcept(false); template (), declval()))> std::true_type can_interprete(int) noexcept(noexcept(interprete(declval(), declval()))); template std::false_type can_encode_data(...) noexcept(false); template (), declval()))> std::true_type can_encode_data(int) noexcept(noexcept(encode_data(declval(), declval()))); template std::false_type can_encode_metadata(...) noexcept(false); template (), declval()))> std::true_type can_encode_metadata(int) noexcept(noexcept(encode_metadata(declval(), declval()))); template std::false_type can_decode(...) noexcept(false); template (), declval(), declval()))> std::true_type can_decode(int) noexcept(noexcept(decode(declval(), declval(), declval()))); } // namespace adl_tests template static constexpr bool is_interpretable() { return decltype(adl_tests::can_interprete::type, A>(0))::value; } template static constexpr bool can_encode_data() { return decltype(adl_tests::can_encode_data::type>(0))::value; } template static constexpr bool can_encode_metadata() { return decltype(adl_tests::can_encode_metadata::type>(0))::value; } template static constexpr bool is_encodable() { return can_encode_data() && can_encode_metadata(); } template static constexpr bool is_decodable() { return decltype(adl_tests::can_decode::type>(0))::value; } template static constexpr bool is_encodable_rvalue() { return is_encodable() && std::is_rvalue_reference::value; }; template using enable_if_t = typename std::enable_if::type; template using disable_if_t = typename std::enable_if::type; template using enable_if_encodable_t = enable_if_t()>; template using disable_if_encodable_t = disable_if_t()>; template using enable_if_decodable_t = enable_if_t()>; template using disable_if_decodable_t = disable_if_t()>; template using enable_if_can_encode_metadata_t = enable_if_t()>; template using disable_if_can_encode_metadata_t = disable_if_t()>; template using enable_if_can_encode_data_t = enable_if_t()>; template using disable_if_can_encode_data_t = disable_if_t()>; template using enable_if_interpretable_t = enable_if_t()>; template using disable_if_interpretable_t = disable_if_t()>; template using enable_if_rvalue_t = enable_if_t::value>; template using enable_if_move_constructible_encodable_rvalue_t = enable_if_t() && std::is_rvalue_reference() && std::is_move_constructible()>; template using enable_if_move_constructible_decodable_rvalue_t = enable_if_t() && std::is_rvalue_reference() && std::is_move_constructible()>; template using enable_if_scalar_t = enable_if_t::value && EnableBool>; template constexpr bool is_array_datatype() { return std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v; } template using enable_if_array_datatype = typename std::enable_if(), int>::type; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/DataInfo.h0000664000175000017500000000326215161702250021050 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/codec/detail/Checksum.h" #include "eckit/codec/detail/Endian.h" namespace eckit::codec { class DataInfo { public: int section() const { return section_; } const std::string& compression() const { return compression_; } Endian endian() const { return endian_; } void endian(Endian e) { endian_ = e; } void compression(const std::string& c) { compression_ = c; } void size(size_t s) { uncompressed_size_ = s; } size_t size() const { return uncompressed_size_; } void compressed_size(size_t s) { compressed_size_ = s; } size_t compressed_size() const { return compressed_size_; } void compressed(bool f) { if (f == false) { compression("none"); } } bool compressed() const { return compression_ != "none"; } explicit operator bool() const { return section_ > 0; } const Checksum& checksum() const { return checksum_; } void checksum(const std::string& c) { checksum_ = Checksum(c); } void section(int s) { section_ = s; } private: int section_{0}; std::string compression_{"none"}; Checksum checksum_; Endian endian_{Endian::native}; size_t uncompressed_size_{0}; size_t compressed_size_{0}; }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Link.cc0000664000175000017500000000217215161702250020415 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/detail/Link.h" #include "eckit/codec/RecordItem.h" #include "eckit/filesystem/PathName.h" namespace eckit::codec { // bool Link::relative() const { // std::string path = RecordItem::URI{uri}.path; // if( path.size() == 0 ) { // return true; // } // if( path[0] == '/' ) { // return false; // } // if( path[0] == '~' ) { // return false; // } // return true; // } // Link operator/( const PathName& dir, const Link& link ) { // auto relative_path = PathName{ RecordItem::URI{link}.path }; // auto absolute_uri = RecordItem::URI(link); // absolute_uri.path = dir / relative_path; // return Link{absolute_uri.str()}; // }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Version.h0000664000175000017500000000311415161702250021004 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/types/SemanticVersion.h" namespace eckit::codec { struct Version { // 8 bytes std::uint32_t major{0}; ///< Major version std::uint32_t minor{2}; ///< Minor version std::string str() const { return std::to_string(major) + "." + std::to_string(minor); } operator std::string() const { return str(); } operator SemanticVersion() const { return SemanticVersion{major, minor, 0}; } bool operator<(const Version& v) const { return SemanticVersion{major, minor, 0} < SemanticVersion{v.major, v.minor, 0}; } bool operator==(const Version& v) const { return SemanticVersion{major, minor, 0} == SemanticVersion{v.major, v.minor, 0}; } bool operator!=(const Version& v) const { return !(*this == v); } bool operator<=(const Version& v) const { return (*this < v) or (*this == v); } bool operator>(const Version& v) const { return !(*this <= v); } bool operator>=(const Version& v) const { return (*this > v) or (*this == v); } friend std::ostream& operator<<(std::ostream& out, const Version& v) { out << v.str(); return out; } }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Endian.h0000664000175000017500000000161415161702250020560 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/eckit_config.h" #ifndef eckit_BIG_ENDIAN #error eckit_BIG_ENDIAN not defined #endif #ifndef eckit_LITTLE_ENDIAN #error eckit_LITTLE_ENDIAN not defined #endif namespace eckit::codec { enum class Endian { little = 0, big = 1, #if eckit_BIG_ENDIAN native = big, swapped = little #elif eckit_LITTLE_ENDIAN native = little, swapped = big #else #error Neither eckit_BIG_ENDIAN nor eckit_LITTLE_ENDIAN equals true #endif }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Type.h0000664000175000017500000000147515161702250020310 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include namespace eckit::codec { struct Type { const std::string name_; explicit operator const std::string&() { return name_; } operator bool() const { return !name_.empty(); } Type(const char* name) : name_(name) {} explicit Type(const std::string& name) : name_(name) {} bool operator==(const Type& other) const { return name_ == other.name_; } }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/ParsedRecord.h0000664000175000017500000000270215161702250021736 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/Metadata.h" #include "eckit/codec/detail/RecordSections.h" namespace eckit::codec { /// Low-level Record information container. /// /// No big data is kept here, only metadata, and information /// on how to retrieve data at a later stage class ParsedRecord { public: RecordHead head; ///< head section of parsed record std::vector keys; ///< Keys of items encoded in parsed record std::map items; ///< Items encoded in parsed record std::vector data_sections; ///< Description of data sections in parsed record /// The parse() function needs to be called during the reading of the record and /// completes the "items" through introspection of the "data_sections". /// It also computes uncompressed data size using available metadata in the "items" void parse(); }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/sfinae.h0000664000175000017500000000577415161702250020642 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/codec/detail/TypeTraits.h" namespace eckit::codec { // ------------------------------------------------------------------------------------------------------- namespace { // These anonymous namespace functions are to avoid recursive behaviour in // following sfinae namespace template = 0> inline void do_interprete(const T& in, A& interpreted) { interprete(in, interpreted); } template = 0> inline size_t do_encode_metadata(const T& in, Metadata& out) { size_t size = encode_metadata(in, out); return size; } template = 0> inline void do_encode_data(const T& in, Data& out) { encode_data(in, out); } } // namespace // ------------------------------------------------------------------------------------------------------- namespace sfinae { // ------------------------------------------------------------------------------------------------------- template = 0> bool interprete(const T& in, A& interpreted) { do_interprete(in, interpreted); return true; } template = 0> bool interprete(const T& /*in*/, A& /*interpreted*/) { return false; } // ------------------------------------------------------------------------------------------------------- template = 0> bool encode_data(const T& in, Data& out) { do_encode_data(in, out); return true; } template = 0> bool encode_data(const T&, Data&) { return false; } // ------------------------------------------------------------------------------------------------------- template = 0> bool encode_metadata(const T& in, Metadata& out) { do_encode_metadata(in, out); return true; } template = 0> bool encode_metadata(const T&, Metadata&) { return false; } template = 0> bool encode_metadata(const T& in, Metadata& out, size_t& data_size) { data_size = do_encode_metadata(in, out); return true; } template = 0> bool encode_metadata(const T&, Metadata&, size_t& data_size) { data_size = 0; return false; } // ------------------------------------------------------------------------------------------------------- } // namespace sfinae } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Checksum.cc0000664000175000017500000000357215161702250021267 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/detail/Checksum.h" #include #include "eckit/codec/detail/Defaults.h" #include "eckit/utils/Hash.h" #include "eckit/utils/Tokenizer.h" namespace eckit::codec { Checksum::Checksum(const std::string& checksum) { std::vector tokens; Tokenizer tokenizer(':'); Tokenizer{':'}(checksum, tokens); if (tokens.size() == 1) { algorithm_ = "none"; checksum_ = ""; } else { algorithm_ = tokens[0]; checksum_ = tokens[1]; } } bool Checksum::available() const { return !checksum_.empty() && algorithm_ != "none"; } std::string Checksum::str() const { if (algorithm_.empty()) { return ""; } return algorithm_ + ":" + checksum_; } std::string Checksum::str(size_t size) const { if (algorithm_.empty()) { return ""; } return algorithm_ + ":" + checksum_.substr(0, std::min(size, checksum_.size())); } std::string checksum(const void* buffer, size_t size, const std::string& algorithm) { auto is_available = [](const std::string& alg) -> bool { return HashFactory::instance().has(alg); }; auto hash = [&](const std::string& alg) -> std::string { std::unique_ptr hasher(HashFactory::instance().build(alg)); return alg + ":" + hasher->compute(buffer, static_cast(size)); }; auto alg = algorithm.empty() ? defaults::checksum_algorithm() : algorithm; return hash(is_available(alg) ? alg : "none"); } } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Base64.cc0000664000175000017500000001071615161702250020547 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/detail/Base64.h" #include namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- namespace { const std::array b64_decode_table{ /* ASCII table */ 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 62, 64, 64, 64, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 64, 64, 64, 64, 64, 64, 64, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 64, 64, 64, 64, 64, 64, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64}; const std::array b64_encode_table{ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; } // namespace //--------------------------------------------------------------------------------------------------------------------- std::string Base64::encode(const void* data, size_t len) { const auto& table = b64_encode_table; const auto* src = reinterpret_cast(data); size_t out_len = 4 * ((len + 2) / 3); /* 3-byte blocks to 4-byte */ if (out_len < len) { return {}; /* integer overflow */ } std::string str; str.resize(out_len); auto* out = reinterpret_cast(const_cast(str.data())); auto* pos = out; const auto* end = src + len; const auto* in = src; while (end - in >= 3) { *pos++ = table[in[0] >> 2]; *pos++ = table[((in[0] & 0x03) << 4) | (in[1] >> 4)]; *pos++ = table[((in[1] & 0x0f) << 2) | (in[2] >> 6)]; *pos++ = table[in[2] & 0x3f]; in += 3; } if (end - in) { *pos++ = table[in[0] >> 2]; if (end - in == 1) { *pos++ = table[(in[0] & 0x03) << 4]; *pos++ = '='; } else { *pos++ = table[((in[0] & 0x03) << 4) | (in[1] >> 4)]; *pos++ = table[(in[1] & 0x0f) << 2]; } *pos++ = '='; } return str; } //--------------------------------------------------------------------------------------------------------------------- std::string Base64::decode(const void* data, size_t len) { const auto& table = b64_decode_table; const auto* p = reinterpret_cast(data); int pad = len > 0 && (len % 4 || p[len - 1] == '='); const size_t L = ((len + 3) / 4 - pad) * 4; std::string str(L / 4 * 3 + pad, '\0'); for (size_t i = 0, j = 0; i < L; i += 4) { int n = table[p[i]] << 18 | table[p[i + 1]] << 12 | table[p[i + 2]] << 6 | table[p[i + 3]]; str[j++] = n >> 16; str[j++] = n >> 8 & 0xFF; str[j++] = n & 0xFF; } if (pad != 0) { int n = table[p[L]] << 18 | table[p[L + 1]] << 12; str[str.size() - 1] = n >> 16; if (len > L + 2 && p[L + 2] != '=') { n |= table[p[L + 2]] << 6; str.push_back(n >> 8 & 0xFF); } } return str; } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/Decoder.h0000664000175000017500000000273015161702250020727 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/codec/Data.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/detail/TypeTraits.h" namespace eckit::codec { class Decoder { public: template = 0> explicit Decoder(T& value) : self_(new DecodableItem(value)) {} friend void decode(const Metadata& metadata, const Data& data, Decoder&); friend void decode(const Metadata& metadata, const Data& data, Decoder&&); private: struct Decodable { virtual ~Decodable() = default; virtual void decode_(const Metadata&, const Data&) = 0; }; template struct DecodableItem : Decodable { explicit DecodableItem(T& value) : data_(value) {} void decode_(const Metadata& metadata, const Data& encoded) override { decode(metadata, encoded, data_); } T& data_; }; std::shared_ptr self_; }; void decode(const Metadata&, const Data&, Decoder&); void decode(const Metadata&, const Data&, Decoder&&); } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/DataType.cc0000664000175000017500000000214715161702250021235 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/detail/DataType.h" #include #include "eckit/codec/Exceptions.h" //------------------------------------------------------------------------------------------------------ namespace eckit::codec { void DataType::throw_not_recognised(kind_t kind) { std::stringstream msg; msg << "kind [" << kind << "] not recognised."; throw Exception(msg.str(), Here()); } void DataType::throw_not_recognised(std::string datatype) { std::stringstream msg; msg << "datatype [" << datatype << "] not recognised."; throw Exception(msg.str(), Here()); } //------------------------------------------------------------------------------------------------------ } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/NoConfig.h0000664000175000017500000000156215161702250021066 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/config/LocalConfiguration.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class NoConfig : public LocalConfiguration { public: NoConfig() = default; virtual ~NoConfig() = default; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/StaticAssert.h0000664000175000017500000000512415161702250021773 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "eckit/codec/eckit_codec_config.h" #if eckit_CODEC_STATIC_ASSERT #include #include "eckit/codec/detail/TypeTraits.h" namespace eckit::codec { template = 0> void encode_data(const T& in, Data& out) { static_assert( can_encode_data(), "\n\n" "\n Static assertion failed" "\n -----------------------" "\n" "\n Values of template type T cannot be encoded into Data because following function is not defined:" "\n" "\n void encode_data(const T& value, Data& out);" "\n" "\n Note that argument-dependent-lookup rules apply." "\n --> The function must be declared in the namespace of type T." "\n\n"); } template = 0> size_t encode_metadata(const T&, Metadata&) { static_assert( can_encode_metadata(), "\n\n" "\n Static assertion failed" "\n -----------------------" "\n" "\n Values of template type T cannot be incoded into Metadata because following function is not defined:" "\n" "\n size_t encode_metadata(const T& value, Metadata& metadata);" "\n" "\n Note that argument-dependent-lookup rules apply." "\n --> The function must be declared in the namespace of type T" "\n\n"); return 0; } template = 0> void decode(const Metadata&, const Data&, T&) { static_assert(is_decodable(), "\n\n" "\n Static assertion failed" "\n -----------------------" "\n" "\n Values of template type T cannot be decoded because following function is not defined:" "\n" "\n void decode(const Metadata&, const Data&, T& out);" "\n" "\n Note that argument-dependent-lookup rules apply." "\n The function must be declared in the namespace of type T" "\n\n"); } } // namespace eckit::codec #endif eckit-2.0.7/src/eckit/codec/detail/Encoder.h0000664000175000017500000000637015161702250020745 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include "eckit/codec/Data.h" #include "eckit/codec/RecordItem.h" #include "eckit/codec/detail/DataInfo.h" #include "eckit/codec/detail/Link.h" #include "eckit/codec/detail/Reference.h" #include "eckit/codec/detail/TypeTraits.h" namespace eckit::codec { class Encoder { public: Encoder() = default; operator bool() const { return static_cast(self_); } template = 0> explicit Encoder(T&& x) : self_(new EncodableValue(std::move(x))) {} explicit Encoder(const Link& link) : self_(new EncodableLink(link)) {} Encoder(Encoder&& other) : self_(std::move(other.self_)) {} template = 0> explicit Encoder(const T& x) : self_(new EncodableValue(x)) {} Encoder& operator=(Encoder&& rhs) { self_ = std::move(rhs.self_); return *this; } friend size_t encode_metadata(const Encoder&, Metadata&); friend void encode_data(const Encoder&, Data&); bool encodes_data() const { return self_->encodes_data_(); } private: struct Encodable { virtual ~Encodable() = default; virtual size_t encode_metadata_(Metadata&) const = 0; virtual void encode_data_(Data&) const = 0; virtual bool encodes_data_() const = 0; }; template struct EncodableValue : Encodable { explicit EncodableValue(Value&& v) : value_{std::move(v)} { sfinae::encode_metadata(value_, metadata_, data_size_); } template = 0> explicit EncodableValue(const Value& v) : value_{v} { sfinae::encode_metadata(value_, metadata_, data_size_); } size_t encode_metadata_(Metadata& metadata) const override { metadata.set(metadata_); return data_size_; } void encode_data_(Data& out) const override { sfinae::encode_data(value_, out); } bool encodes_data_() const override { return data_size_ > 0; } const Value value_; Metadata metadata_; size_t data_size_{0}; }; struct EncodableLink : Encodable { explicit EncodableLink(const Link& link) : link_(link) {} size_t encode_metadata_(Metadata& metadata) const override { metadata.set(Metadata("link", link_.uri)); return 0; } void encode_data_(Data& /*out*/) const override {} bool encodes_data_() const override { return false; } Link link_; }; std::unique_ptr self_; }; size_t encode_metadata(const Encoder& encoder, Metadata& metadata); void encode_data(const Encoder& encoder, Data& out); } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/detail/demangle.h0000664000175000017500000000155015161702250021135 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- std::string demangle(const char*); template std::string demangle() { return demangle(typeid(T).name()); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/eckit_codec_config.h.in0000664000175000017500000000070415161702250022305 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #cmakedefine01 eckit_CODEC_STATIC_ASSERT eckit-2.0.7/src/eckit/codec/Exceptions.h0000664000175000017500000000507615161702250020247 0ustar alastairalastair/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/detail/demangle.h" #include "eckit/exception/Exceptions.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class Exception : public eckit::Exception { public: using eckit::Exception::Exception; ~Exception() override; }; //--------------------------------------------------------------------------------------------------------------------- class NotEncodable : Exception { public: NotEncodable(const std::string& type_name); template NotEncodable(const T&) : NotEncodable{demangle::type>()} {} ~NotEncodable() override; }; //--------------------------------------------------------------------------------------------------------------------- class NotDecodable : public Exception { public: NotDecodable(const std::string& type_name); template NotDecodable(const T&) : NotDecodable{demangle::type>()} {} ~NotDecodable() override; }; //--------------------------------------------------------------------------------------------------------------------- class InvalidRecord : public Exception { public: InvalidRecord(const std::string& message) : Exception("eckit::codec::InvalidRecord: " + message) {} ~InvalidRecord() override; }; //--------------------------------------------------------------------------------------------------------------------- class DataCorruption : public Exception { public: DataCorruption(const std::string& message) : Exception("eckit::codec::DataCorruption: " + message) {} ~DataCorruption() override; }; //--------------------------------------------------------------------------------------------------------------------- class WriteError : public Exception { public: WriteError(const std::string& message) : Exception("eckit::codec::WriteError: " + message) {} ~WriteError() override; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/RecordItemReader.cc0000664000175000017500000001636515161702250021447 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/RecordItemReader.h" #include "eckit/codec/Exceptions.h" #include "eckit/codec/FileStream.h" #include "eckit/codec/Record.h" #include "eckit/codec/Session.h" #include "eckit/codec/detail/ParsedRecord.h" #include "eckit/codec/detail/RecordSections.h" namespace eckit::codec { namespace { //--------------------------------------------------------------------------------------------------------------------- template inline size_t read_struct(IStream& in, Struct& s) { static_assert(Struct::bytes == sizeof(Struct)); return in.read(reinterpret_cast(&s), sizeof(Struct)); } //--------------------------------------------------------------------------------------------------------------------- template inline Struct read_struct(IStream& in) { Struct s; if (read_struct(in, s) != sizeof(Struct)) { throw InvalidRecord("Unexpected EOF reached"); } return s; } //--------------------------------------------------------------------------------------------------------------------- Data read_data(const Record& record, int data_section_index, Stream in) { if (data_section_index == 0) { return {}; } const auto& parsed = static_cast(record); const auto& data_section = parsed.data_sections.at(static_cast(data_section_index) - 1); Data data; auto offset = data_section.offset; in.seek(offset); auto data_begin = read_struct(in); if (not data_begin.valid()) { throw InvalidRecord("Data section is not valid"); } auto data_size = static_cast(data_section.length) - sizeof(RecordDataSection::Begin) - sizeof(RecordDataSection::End); if (data_size > 0) { if (data.read(in, data_size) != data_size) { throw InvalidRecord("Data section is not valid"); } ASSERT(data.size() == data_size); } auto data_end = read_struct(in); if (not data_end.valid()) { throw InvalidRecord("Data section is not valid"); } return data; } //--------------------------------------------------------------------------------------------------------------------- PathName make_absolute_path(const std::string& reference_path, RecordItem::URI& uri) { PathName absolute_path = uri.path; if (!reference_path.empty() && uri.path[0] != '/' && uri.path[0] != '~') { absolute_path = PathName{reference_path} / absolute_path; } return absolute_path.fullName(); } //--------------------------------------------------------------------------------------------------------------------- Record read_record(const std::string& path, size_t offset) { auto record = Session::record(path, offset); if (record.empty()) { auto in = InputFileStream(path); in.seek(offset); record.read(in); } return record; } //--------------------------------------------------------------------------------------------------------------------- Record read_record(Stream in, size_t offset) { auto record = Session::record(in, offset); if (record.empty()) { in.seek(offset); record.read(in); } return record; } //--------------------------------------------------------------------------------------------------------------------- } // anonymous namespace //--------------------------------------------------------------------------------------------------------------------- RecordItemReader::RecordItemReader(Stream in, size_t offset, const std::string& key) : in_(in), uri_{"", offset, key} { record_ = read_record(in, uri_.offset); if (not record_.has(uri_.key)) { throw InvalidRecord(uri_.key + " not found in record " + uri_.path); } } RecordItemReader::RecordItemReader(Stream in, const std::string& key) : in_(in), uri_{"", 0, key} { record_ = read_record(in, uri_.offset); if (not record_.has(uri_.key)) { throw InvalidRecord(uri_.key + " not found in record " + uri_.path); } } //--------------------------------------------------------------------------------------------------------------------- RecordItemReader::RecordItemReader(const std::string& uri) : RecordItemReader("", uri) {} //--------------------------------------------------------------------------------------------------------------------- RecordItemReader::RecordItemReader(const std::string& ref, const std::string& uri) : ref_{ref}, uri_{uri} { auto absolute_path = make_absolute_path(ref_, uri_); if (not absolute_path.exists()) { throw InvalidRecord("Item " + uri_.str() + " refers to non existing file: " + absolute_path); } record_ = read_record(absolute_path, uri_.offset); if (not record_.has(uri_.key)) { throw InvalidRecord(uri_.key + " not found in record " + uri_.path); } } //--------------------------------------------------------------------------------------------------------------------- void RecordItemReader::read(RecordItem& item) { Metadata metadata; Data data; read(metadata, data); item.metadata(metadata); item.data(std::move(data)); }; //--------------------------------------------------------------------------------------------------------------------- void RecordItemReader::read(Metadata& metadata, bool follow_links) { metadata = record_.metadata(uri_.key); if (follow_links && metadata.link()) { auto absolute_path = make_absolute_path(ref_, uri_); Metadata linked; RecordItemReader{absolute_path.dirName(), metadata.link()}.read(linked); metadata.link(std::move(linked)); } }; //--------------------------------------------------------------------------------------------------------------------- static void read_from_stream(Record record, Stream in, const std::string& key, Metadata& metadata, Data& data) { metadata = record.metadata(key); if (metadata.link()) { throw Exception("Cannot follow links in records that are not file based"); } if (metadata.data.section() != 0) { data = read_data(record, metadata.data.section(), in); } } void RecordItemReader::read(Metadata& metadata, Data& data) { if (in_) { read_from_stream(record_, in_, uri_.key, metadata, data); return; } metadata = record_.metadata(uri_.key); auto absolute_path = make_absolute_path(ref_, uri_); if (metadata.link()) { Metadata linked; RecordItemReader{absolute_path.dirName(), metadata.link()}.read(linked, data); metadata.link(std::move(linked)); } else if (metadata.data.section() != 0) { data = read_data(record_, metadata.data.section(), InputFileStream(absolute_path)); } }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/RecordItem.h0000664000175000017500000000523715161702250020162 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/Data.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/detail/tag.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- struct RecordItem { public: struct URI { explicit URI(const std::string& uri); URI(const std::string& _path, std::uint64_t _offset, const std::string& _key); std::string str() const; operator std::string() const { return str(); } std::string path; std::uint64_t offset; std::string key; }; public: RecordItem() = default; template explicit RecordItem(T&& x, tag::enable_static_assert = tag::enable_static_assert()); template explicit RecordItem(T&& x, tag::disable_static_assert); RecordItem(Metadata&&, Data&&); RecordItem(RecordItem&& other); const Data& data() const; const Metadata& metadata() const; void metadata(const Metadata& m); void data(Data&& d); bool empty() const; void clear(); void decompress(); void compress(); private: std::unique_ptr metadata_{new Metadata()}; Data data_; }; //--------------------------------------------------------------------------------------------------------------------- template RecordItem::RecordItem(T&& x, tag::enable_static_assert) { encode(x, *metadata_, data_); } //--------------------------------------------------------------------------------------------------------------------- template RecordItem::RecordItem(T&& x, tag::disable_static_assert) { encode(x, *metadata_, data_, tag::disable_static_assert()); } //--------------------------------------------------------------------------------------------------------------------- size_t encode_metadata(const RecordItem& in, Metadata& metadata); //--------------------------------------------------------------------------------------------------------------------- void encode_data(const RecordItem& in, Data& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Session.cc0000664000175000017500000001226115161702250017701 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/Session.h" #include #include #include #include #include #include #include "eckit/codec/Exceptions.h" #include "eckit/filesystem/PathName.h" namespace eckit::codec { using lock_guard = std::lock_guard; class SessionImpl { public: void store(Stream stream); Record record(const std::string& path, size_t offset); private: std::recursive_mutex mutex_; std::vector handles_; std::map records_; }; //--------------------------------------------------------------------------------------------------------------------- class ActiveSession { public: static ActiveSession& instance(); SessionImpl& current(); void push(); void pop(); Record record(const std::string& path, size_t offset); void store(Stream stream); private: friend class Session; std::recursive_mutex mutex_; std::unique_ptr session_; std::atomic count_{0}; }; //--------------------------------------------------------------------------------------------------------------------- ActiveSession& ActiveSession::instance() { static ActiveSession instance; return instance; } //--------------------------------------------------------------------------------------------------------------------- Record ActiveSession::record(const std::string& path, size_t offset) { if (count_ != 0) { return current().record(path, offset); } return {}; } //--------------------------------------------------------------------------------------------------------------------- void ActiveSession::store(Stream stream) { if (count_ != 0) { return current().store(stream); } } //--------------------------------------------------------------------------------------------------------------------- SessionImpl& ActiveSession::current() { lock_guard lock(mutex_); if (count_ == 0) { throw Exception("No eckit::codec session is currently active", Here()); } return *session_; } //--------------------------------------------------------------------------------------------------------------------- void ActiveSession::push() { lock_guard lock(mutex_); if (count_ == 0) { ASSERT(session_ == nullptr); session_ = std::make_unique(); } ++count_; } //--------------------------------------------------------------------------------------------------------------------- void ActiveSession::pop() { lock_guard lock(mutex_); if (count_ == 0) { throw Exception("No eckit::codec session is currently active", Here()); } --count_; if (count_ == 0) { session_.reset(); } } //--------------------------------------------------------------------------------------------------------------------- void SessionImpl::store(Stream stream) { lock_guard lock(mutex_); handles_.emplace_back(stream); } //--------------------------------------------------------------------------------------------------------------------- Record SessionImpl::record(const std::string& path, size_t offset) { lock_guard lock(mutex_); auto key = Record::URI{PathName(path).fullName(), offset}.str(); if (records_.find(key) == records_.end()) { records_.emplace(key, Record{}); } return records_.at(key); } //--------------------------------------------------------------------------------------------------------------------- Session::Session() { ActiveSession::instance().push(); } //--------------------------------------------------------------------------------------------------------------------- Session::~Session() { ActiveSession::instance().pop(); } //--------------------------------------------------------------------------------------------------------------------- bool Session::active() { return ActiveSession::instance().count_ > 0; } //--------------------------------------------------------------------------------------------------------------------- Record Session::record(const std::string& path, size_t offset) { return ActiveSession::instance().record(path, offset); } //--------------------------------------------------------------------------------------------------------------------- Record Session::record(Stream stream, size_t offset) { std::stringstream id; id << &stream.datahandle(); return ActiveSession::instance().record(id.str(), offset); } //--------------------------------------------------------------------------------------------------------------------- void Session::store(Stream stream) { ActiveSession::instance().store(stream); } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/FileStream.h0000664000175000017500000000346115161702250020155 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/codec/Stream.h" #include "eckit/filesystem/PathName.h" namespace eckit { class DataHandle; } namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- enum class Mode { read, append, write, }; //--------------------------------------------------------------------------------------------------------------------- class FileStream : public Stream { public: FileStream(const PathName&, Mode openmode); FileStream(const PathName&, char openmode); FileStream(const PathName&, const std::string& openmode); }; //--------------------------------------------------------------------------------------------------------------------- class InputFileStream : public FileStream { public: explicit InputFileStream(const PathName&); }; //--------------------------------------------------------------------------------------------------------------------- class OutputFileStream : public FileStream { public: explicit OutputFileStream(const PathName&, Mode openmode = Mode::write); OutputFileStream(const PathName&, const std::string& openmode); OutputFileStream(const PathName&, char openmode); void close(); }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/print/0000775000175000017500000000000015161702250017101 5ustar alastairalastaireckit-2.0.7/src/eckit/codec/print/TableFormat.h0000664000175000017500000000272115161702250021454 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include #include #include "eckit/codec/Metadata.h" #include "eckit/codec/Record.h" #include "eckit/codec/RecordItemReader.h" #include "eckit/codec/Session.h" namespace eckit::codec { class MetadataPrettyPrintBase { public: virtual ~MetadataPrettyPrintBase() = default; virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream&, const MetadataPrettyPrintBase&); std::string str() const; }; class MetadataPrettyPrint { public: explicit MetadataPrettyPrint(const Metadata&); friend std::ostream& operator<<(std::ostream& out, const MetadataPrettyPrint& p); std::string str() const; private: std::unique_ptr impl_; }; class TableFormat { public: TableFormat(const Record::URI& record, const Parametrisation& config); void print(std::ostream&) const; private: const Record record_; std::map items_; bool print_details_{false}; }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/print/TableFormat.cc0000664000175000017500000002264715161702250021623 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/print/TableFormat.h" #include #include "eckit/codec/Exceptions.h" #include "eckit/codec/Metadata.h" #include "eckit/codec/Record.h" #include "eckit/codec/RecordItemReader.h" #include "eckit/codec/Session.h" #include "eckit/codec/print/Bytes.h" #include "eckit/codec/types/array.h" #include "eckit/codec/types/scalar.h" namespace eckit::codec { std::ostream& operator<<(std::ostream& out, const MetadataPrettyPrintBase& p) { p.print(out); return out; } std::string MetadataPrettyPrintBase::str() const { std::stringstream s; print(s); return s.str(); } class DefaultMetadataPrettyPrint : public MetadataPrettyPrintBase { public: explicit DefaultMetadataPrettyPrint(const Metadata&) {} void print(std::ostream&) const override {} }; class ArrayMetadataPrettyPrint : public MetadataPrettyPrintBase { template void print_value(std::ostream& out) const { std::vector value; metadata_.get("value", value); out << "{"; for (size_t i = 0; i < value.size(); ++i) { out << value[i]; if (i < value.size() - 1) { out << ","; } } out << "}"; } public: explicit ArrayMetadataPrettyPrint(const Metadata& m) : metadata_(m) {} void print(std::ostream& out) const override { std::string type = metadata_.getString("type"); ASSERT(type == "array"); ArrayMetadata array(metadata_); out << std::setw(7) << std::left << array.datatype().str(); if (metadata_.has("value")) { out << ": "; std::string datatype = metadata_.getString("datatype"); if (datatype == DataType::str()) { print_value(out); } else if (datatype == DataType::str()) { print_value(out); } else if (datatype == DataType::str()) { print_value(out); } else if (datatype == DataType::str()) { print_value(out); } else if (datatype == DataType::str()) { print_value(out); } } else { out << "["; for (int i = 0; i < array.rank(); ++i) { out << array.shape(i); if (i < array.rank() - 1) { out << ","; } } out << "]"; } } private: Metadata metadata_; }; class StringMetadataPrettyPrint : public MetadataPrettyPrintBase { public: explicit StringMetadataPrettyPrint(const Metadata& m) : metadata_(m) {} void print(std::ostream& out) const override { std::string type = metadata_.getString("type"); ASSERT(type == "string"); std::string value = metadata_.getString("value"); if (value.size() <= 32) { out << value; } else { out << value.substr(0, 32) << "..."; } } private: Metadata metadata_; }; class ScalarMetadataPrettyPrint : public MetadataPrettyPrintBase { public: explicit ScalarMetadataPrettyPrint(const Metadata& m) : metadata_(m) {} template T decode() const { T value; Data dummy; codec::decode(metadata_, dummy, value); return value; } void print(std::ostream& out) const override { std::string type = metadata_.getString("type"); ASSERT(type == "scalar"); std::string datatype = metadata_.getString("datatype"); out << std::setw(7) << std::left << datatype << ": "; if (datatype == DataType::str()) { out << decode(); } else if (datatype == DataType::str()) { out << decode(); } else if (datatype == DataType::str()) { out << decode(); } else if (datatype == DataType::str()) { out << decode(); } else if (datatype == DataType::str()) { out << decode(); } } private: Metadata metadata_; }; MetadataPrettyPrint::MetadataPrettyPrint(const Metadata& m) { std::string type = m.getString("type"); // Poor man factory builder if (type == "array") { impl_.reset(new ArrayMetadataPrettyPrint(m)); } else if (type == "scalar") { impl_.reset(new ScalarMetadataPrettyPrint(m)); } else if (type == "string") { impl_.reset(new StringMetadataPrettyPrint(m)); } else { impl_.reset(new DefaultMetadataPrettyPrint(m)); } } std::ostream& operator<<(std::ostream& out, const MetadataPrettyPrint& p) { out << *p.impl_; return out; } std::string MetadataPrettyPrint::str() const { return impl_->str(); } struct TablePrinter { std::vector > columns; std::vector widths; std::string indent{" "}; std::string sep{" "}; int col{0}; int row{1}; std::vector optional; std::vector underline; TablePrinter() = default; TablePrinter& column(const std::string& title, size_t width = 0) { columns.emplace_back(std::vector{title}); widths.emplace_back(std::max(title.size(), width)); optional.push_back(false); underline.push_back(true); return *this; } TablePrinter& column() { columns.emplace_back(std::vector{""}); widths.emplace_back(0); optional.push_back(true); underline.push_back(false); return *this; } TablePrinter& operator<<(const MetadataPrettyPrint p) { *this << p.str(); return *this; } TablePrinter& operator<<(const std::string& s) { columns[col].emplace_back(s); widths[col] = std::max(widths[col], s.size()); if (optional[col] && widths[col] > 0) { optional[col] = false; widths[col] = std::max(widths[col], columns[col][0].size()); } ++col; if (col == columns.size()) { ++row; col = 0; } return *this; } void print(std::ostream& out) const { out << " "; for (size_t c = 0; c < columns.size(); ++c) { out << " " << sep << " " << std::setw(widths[c]) << std::left << columns[c][0]; } out << " " << sep << std::endl; out << " "; for (size_t c = 0; c < columns.size(); ++c) { const char u = underline[c] ? '-' : ' '; out << " " << sep << " " << std::string(widths[c], u); // underline } out << " " << sep << std::endl; for (size_t r = 1; r < row; ++r) { out << " "; for (size_t c = 0; c < columns.size(); ++c) { out << " " << sep << " " << std::setw(widths[c]) << std::left << columns[c][r]; } out << " " << sep << std::endl; } } }; TableFormat::TableFormat(const Record::URI& record, const Parametrisation& config) : record_(Session::record(record.path, record.offset)) { for (const auto& key : record_.keys()) { items_.emplace(key, Metadata()); RecordItemReader{RecordItem::URI{record.path, record.offset, key}}.read(items_.at(key)); } config.get("details", print_details_); } void TableFormat::print(std::ostream& out) const { ASSERT(not record_.empty()); TablePrinter table; table.column("name"); table.column("type"); table.column("description"); if (print_details_) { table.column("ver."); table.column("comp."); table.column("size"); table.column("checksum[:8]"); table.column("endian"); table.column("created"); } table.column(); for (const auto& key : record_.keys()) { const auto& item = items_.at(key); size_t cbytes = item.data.compressed_size(); size_t ubytes = item.data.size(); std::string endian = (item.data.endian() == Endian::little) ? "little" : "big"; std::string created = item.record.created().str().substr(0, 16); created[10] = ' '; table << key; table << item.getString("type"); table << MetadataPrettyPrint{item}; if (print_details_) { table << item.record.version(); if (item.data.section() != 0) { table << item.data.compression(); table << Bytes{cbytes}.str() + (item.data.compressed() ? " < " + Bytes{ubytes}.str() : ""); table << Checksum{item.data.checksum()}.str(8); table << endian; } else { table << "" << "" << "" << ""; } table << created; } if (record_.metadata(key).link()) { table << "-> " + record_.metadata(key).link().str(); } else { table << ""; } } table.print(out); } } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/print/JSONFormat.cc0000664000175000017500000000427315161702250021340 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/print/JSONFormat.h" #include "eckit/codec/Record.h" #include "eckit/codec/RecordItemReader.h" #include "eckit/codec/Session.h" #include "eckit/log/JSON.h" namespace eckit::codec { JSONFormat::JSONFormat(const Record::URI& record, const Configuration& config) : record_(Session::record(record.path, record.offset)) { for (const auto& key : record_.keys()) { items_.emplace(key, Metadata()); RecordItemReader{RecordItem::URI{record.path, record.offset, key}}.read(items_.at(key)); } config.get("details", print_details_); } void JSONFormat::print(std::ostream& out) const { JSON js(out, JSON::Formatting::indent(4)); Metadata metadata; for (const auto& key : record_.keys()) { const auto& item = items_.at(key); Metadata m = record_.metadata(key); if (record_.metadata(key).link()) { m.set(item); if (m.has("data")) { // removes data.section m.remove("data"); } } if (not print_details_) { if (m.has("data")) { m.remove("data"); } } if (print_details_) { if (item.data.size() > 0) { m.set("data.compression.type", item.data.compression()); m.set("data.compression.size", item.data.compressed_size()); m.set("data.size", item.data.size()); m.set("data.byte_order", (item.data.endian() == Endian::little) ? "little endian" : "big endian"); m.set("data.checksum", item.data.checksum().str()); } m.set("version", item.record.version().str()); m.set("created", item.record.created().str()); } metadata.set(key, m); } js << metadata; } } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/print/Bytes.h0000664000175000017500000000157415161702250020347 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include namespace eckit::codec { struct Bytes { public: explicit Bytes(size_t bytes) : bytes_(bytes) {} explicit operator size_t() const { return bytes_; } std::string str(int decimals = 2, int width = 7) const; void print(std::ostream& out, int decimals = 2, int width = 7) const; friend std::ostream& operator<<(std::ostream&, const Bytes&); private: size_t bytes_; }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/print/JSONFormat.h0000664000175000017500000000155415161702250021201 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include #include "eckit/codec/Metadata.h" #include "eckit/codec/Record.h" #include "eckit/config/Configuration.h" namespace eckit::codec { class JSONFormat { public: JSONFormat(const Record::URI& record, const Configuration&); void print(std::ostream&) const; private: const Record record_; std::map items_; bool print_details_{false}; }; } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/print/Bytes.cc0000664000175000017500000000562115161702250020502 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/print/Bytes.h" #include #include #include #include #include namespace eckit::codec { namespace { template class FixedFormat { public: using value_type = Value; FixedFormat(value_type x, long precision) : x_(x), precision_(precision > 0 ? precision : 20) {} void print(std::ostream& out) const { for (long precision = 0; precision <= precision_; ++precision) { if (is_precision(precision) || precision == precision_) { out << std::setprecision(precision); out << std::fixed << x_; break; } } } bool is_precision(long precision) const { std::stringstream ss; ss << std::setprecision(precision); ss << std::fixed << x_; value_type _x; ss >> _x; return std::abs(x_ - _x) < 1.e-20; } friend std::ostream& operator<<(std::ostream& out, const FixedFormat& This) { This.print(out); return out; } private: value_type x_; long precision_; }; inline FixedFormat fixed_format(double x, long precision) { return {x, precision}; } /* static std::pair reduce_to_10( size_t bytes ) { static const std::vector magnitudes{"B", "K", "M", "G", "T", "P", "E", "Z", "Y"}; double x = bytes; size_t n = 0; while ( x >= 10 && n < magnitudes.size() ) { x /= 1024.; n++; } return std::make_pair( x, magnitudes[n] ); } */ std::pair reduce_to_1000(size_t bytes) { static const std::vector magnitudes{"B", "K", "M", "G", "T", "P", "E", "Z", "Y"}; double x = bytes; size_t n = 0; while (x >= 1000 && n < magnitudes.size()) { x /= 1024.; n++; } return std::make_pair(x, magnitudes[n]); } } // namespace void Bytes::print(std::ostream& out, int decimals, int width) const { if (bytes_ < 1000 && width >= 4) { out << std::setw(width - 1) << std::right << bytes_ << 'B'; } else { auto pair = reduce_to_1000(bytes_); out << std::setw(width - 1) << std::right << fixed_format(pair.first, decimals); out << pair.second; } } std::ostream& operator<<(std::ostream& out, const Bytes& bytes) { bytes.print(out); return out; } std::string Bytes::str(int decimals, int width) const { std::stringstream s; print(s, decimals, width); return s.str(); } } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/RecordPrinter.cc0000664000175000017500000000656415161702250021051 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/RecordPrinter.h" #include #include "eckit/codec/Exceptions.h" #include "eckit/codec/FileStream.h" #include "eckit/codec/print/JSONFormat.h" #include "eckit/codec/print/TableFormat.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- RecordPrinter::RecordPrinter(const PathName& path, const Configuration& config) : RecordPrinter(path, 0, config) {} //--------------------------------------------------------------------------------------------------------------------- RecordPrinter::RecordPrinter(const PathName& path, const std::uint64_t offset, const Configuration& config) : RecordPrinter(Record::URI{path, offset}, config) {} //--------------------------------------------------------------------------------------------------------------------- RecordPrinter::RecordPrinter(const Record::URI& ref, const Configuration& config) : uri_(ref), record_(Session::record(ref.path, ref.offset)) { if (record_.empty()) { auto in = InputFileStream(uri_.path); in.seek(uri_.offset); record_.read(in, true); ASSERT(not record_.empty()); } config.get("format", options_.format); config.get("details", options_.details); // Check if format is supported { std::vector supported_formats{"json", "yaml", "table"}; bool format_supported{false}; for (auto& supported_format : supported_formats) { if (options_.format == supported_format) { format_supported = true; break; } } if (not format_supported) { std::stringstream s; s << "Format '" + options_.format + "' not supported. Supported formats:"; for (auto& supported_format : supported_formats) { s << "\n - " << supported_format; } throw Exception(s.str(), Here()); } } } //--------------------------------------------------------------------------------------------------------------------- void RecordPrinter::print(std::ostream& out) const { LocalConfiguration config; config.set("details", options_.details); if (options_.format == "json") { JSONFormat{uri_, config}.print(out); } else if (options_.format == "yaml") { JSONFormat{uri_, config}.print(out); } else if (options_.format == "table") { TableFormat{uri_, config}.print(out); } else { throw Exception("Cannot print record: Unrecognized format " + options_.format + ".", Here()); } } //--------------------------------------------------------------------------------------------------------------------- std::ostream& operator<<(std::ostream& out, const RecordPrinter& info) { info.print(out); return out; } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Data.h0000664000175000017500000000321415161702250016767 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/io/Buffer.h" namespace eckit::codec { class Stream; //--------------------------------------------------------------------------------------------------------------------- class Data { public: Data() = default; Data(void*, size_t); Data(Data&&) = default; Data& operator=(Data&&) = default; operator const void*() const { return data(); } const void* data() const { return buffer_.data(); } size_t size() const { return size_; } void assign(const Data& other); void assign(const void*, size_t); void clear(); std::uint64_t write(Stream& out) const; std::uint64_t read(Stream& in, size_t size); void compress(const std::string& compression); void decompress(const std::string& compression, size_t uncompressed_size); std::string checksum(const std::string& algorithm = "") const; private: Buffer buffer_; size_t size_{0}; }; //--------------------------------------------------------------------------------------------------------------------- void encode_data(const Data&, Data& out); //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/RecordItemReader.h0000664000175000017500000000255415161702250021304 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/codec/Record.h" #include "eckit/codec/RecordItem.h" #include "eckit/codec/Stream.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- class RecordItemReader { public: RecordItemReader(Stream, size_t offset, const std::string& key); RecordItemReader(Stream, const std::string& key); explicit RecordItemReader(const std::string& uri); void read(RecordItem& item); void read(Metadata&, bool follow_links = true); void read(Metadata&, Data&); private: RecordItemReader(const std::string& ref, const std::string& uri); Stream in_; Record record_; std::string ref_{}; // directory to which relative URI's are evaluated RecordItem::URI uri_; }; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/Exceptions.cc0000664000175000017500000000601515161702250020377 0ustar alastairalastair/* * (C) Copyright 2020 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "Exceptions.h" #include "eckit/codec/detail/demangle.h" namespace eckit::codec { //--------------------------------------------------------------------------------------------------------------------- NotEncodable::NotEncodable(const std::string& type_name) : Exception{[&type_name] { std::stringstream message; message << "eckit::codec::NotEncodable: Cannot encode values of type " << type_name << "."; message << "\n Implement the functions" "\n" "\n void encode_data(const " << type_name << "&, eckit::codec::Data& );" "\n size_t encode_metadata(const " << type_name << "&, eckit::codec::Metadata& );" "\n" "\n or alternatively a conversion function to eckit::codec::types::array::ArrayReference" "\n" "\n void interprete(const " << type_name << "&, eckit::codec::types::array::ArrayReference& )" "\n" "\n Rules of argument-dependent-lookup apply." "\n --> Functions need to be declared in namespace of any of the arguments."; return message.str(); }()} {} //--------------------------------------------------------------------------------------------------------------------- NotDecodable::NotDecodable(const std::string& type_name) : Exception{[&type_name] { std::stringstream message; message << "eckit::codec::NotDecodable: Cannot decode values of type " << type_name << "."; message << "\n Implement the functions" "\n" "\n void decode( const atlas::io::Metadata&, const atlas::io::Data&, " << type_name << "& );" "\n" "\n Rules of argument-dependent-lookup apply." "\n --> Functions need to be declared in namespace of any of the arguments."; return message.str(); }()} {} //--------------------------------------------------------------------------------------------------------------------- Exception::~Exception() = default; NotEncodable::~NotEncodable() = default; NotDecodable::~NotDecodable() = default; InvalidRecord::~InvalidRecord() = default; DataCorruption::~DataCorruption() = default; WriteError::~WriteError() = default; //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/codec/CMakeLists.txt0000664000175000017500000000472315161702250020513 0ustar alastairalastair configure_file( eckit_codec_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/eckit_codec_config.h @ONLY ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/eckit_codec_config.h DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/codec ) ecbuild_add_library( TARGET eckit_codec INSTALL_HEADERS ALL HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/codec PUBLIC_LIBS eckit PUBLIC_INCLUDES $ $ SOURCES ${CMAKE_CURRENT_BINARY_DIR}/eckit_codec_config.h Data.cc Data.h Exceptions.cc Exceptions.h FileStream.cc FileStream.h Metadata.cc Metadata.h ReadRequest.cc ReadRequest.h Record.cc Record.h RecordItem.cc RecordItem.h RecordItemReader.cc RecordItemReader.h RecordPrinter.cc RecordPrinter.h RecordReader.cc RecordReader.h RecordWriter.cc RecordWriter.h Session.cc Session.h Stream.cc Stream.h codec.h detail/Base64.cc detail/Base64.h detail/Checksum.cc detail/Checksum.h detail/DataInfo.h detail/DataType.cc detail/DataType.h detail/Decoder.cc detail/Decoder.h detail/Defaults.h detail/demangle.cc detail/demangle.h detail/Encoder.cc detail/Encoder.h detail/Endian.h detail/Link.cc detail/Link.h detail/NoConfig.h detail/ParsedRecord.h detail/RecordInfo.h detail/RecordSections.h detail/Reference.h detail/StaticAssert.h detail/Time.cc detail/Time.h detail/Type.h detail/TypeTraits.h detail/Version.h detail/sfinae.h detail/tag.h print/Bytes.cc print/Bytes.h print/JSONFormat.cc print/JSONFormat.h print/TableFormat.cc print/TableFormat.h types/array.h types/array/ArrayMetadata.cc types/array/ArrayMetadata.h types/array/ArrayReference.cc types/array/ArrayReference.h types/array/adaptors/StdArrayAdaptor.h types/array/adaptors/StdVectorAdaptor.h types/array/adaptors/StdVectorOfStdArrayAdaptor.h types/scalar.cc types/scalar.h types/string.h ) target_compile_features( eckit_codec PUBLIC cxx_std_17 ) eckit-2.0.7/src/eckit/codec/ReadRequest.cc0000664000175000017500000001051715161702250020504 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/codec/ReadRequest.h" #include "eckit/codec/Exceptions.h" #include "eckit/codec/RecordItemReader.h" #include "eckit/codec/detail/Checksum.h" #include "eckit/codec/detail/Defaults.h" #include "eckit/log/Log.h" namespace eckit::codec { static std::string stream_path(Stream stream) { std::stringstream s; s << &stream.datahandle(); return s.str(); } //--------------------------------------------------------------------------------------------------------------------- ReadRequest::ReadRequest(const std::string& URI, Decoder* decoder) : uri_(URI), decoder_(decoder), item_(new RecordItem()) { do_checksum_ = defaults::checksum_read(); ASSERT(!uri_.empty()); } ReadRequest::ReadRequest(Stream stream, size_t offset, const std::string& key, Decoder* decoder) : stream_{stream}, offset_{offset}, key_{key}, uri_{"stream:" + stream_path(stream) + "?offset=key=" + key_}, decoder_(decoder), item_(new RecordItem()) { do_checksum_ = defaults::checksum_read(); ASSERT(stream_); } //--------------------------------------------------------------------------------------------------------------------- ReadRequest::ReadRequest(ReadRequest&& other) : stream_{other.stream_}, offset_{other.offset_}, key_{other.key_}, uri_(std::move(other.uri_)), decoder_(std::move(other.decoder_)), item_(std::move(other.item_)), do_checksum_{other.do_checksum_}, finished_{other.finished_} { other.do_checksum_ = true; other.finished_ = true; } //--------------------------------------------------------------------------------------------------------------------- ReadRequest::~ReadRequest() { if (item_) { if (not finished_) { Log::error() << "Request for " << uri_ << " was not completed." << std::endl; } } } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::read() { if (item_->empty()) { if (stream_) { RecordItemReader{stream_, offset_, key_}.read(*item_); } else { RecordItemReader(uri_).read(*item_); } } } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::checksum(bool b) { do_checksum_ = b; } void ReadRequest::checksum() { if (not do_checksum_) { return; } Checksum encoded_checksum{item_->metadata().data.checksum()}; if (not encoded_checksum.available()) { return; } Checksum computed_checksum{item_->data().checksum(encoded_checksum.algorithm())}; if (computed_checksum.available() && (computed_checksum.str() != encoded_checksum.str())) { std::stringstream err; err << "Mismatch in checksums for " << uri_ << ".\n"; err << " Encoded: [" << encoded_checksum.str() << "].\n"; err << " Computed: [" << computed_checksum.str() << "]."; throw DataCorruption(err.str()); } do_checksum_ = false; } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::decompress() { read(); item_->decompress(); } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::decode() { decompress(); codec::decode(item_->metadata(), item_->data(), *decoder_); item_->clear(); } //--------------------------------------------------------------------------------------------------------------------- void ReadRequest::wait() { if (item_) { if (not finished_) { read(); checksum(); decompress(); decode(); } finished_ = true; } } //--------------------------------------------------------------------------------------------------------------------- } // namespace eckit::codec eckit-2.0.7/src/eckit/bases/0000775000175000017500000000000015161702250015765 5ustar alastairalastaireckit-2.0.7/src/eckit/bases/Watcher.cc0000664000175000017500000000171315161702250017673 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/bases/Watcher.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- struct DummyWatcher : public Watcher { void watch() {} }; Watcher& Watcher::dummy() { static DummyWatcher x; return x; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/bases/Loader.h0000664000175000017500000000211015161702250017336 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Loader.h // Baudouin Raoult - ECMWF Jul 96 #ifndef eckit_Loader_h #define eckit_Loader_h #include "eckit/eckit.h" #include "eckit/container/ClassExtent.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class Loader : public ClassExtent { public: // methods Loader(); virtual ~Loader(); virtual void execute() = 0; private: // methods // There is no private copy constructor as this will confuse g++ 4.x.x }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/bases/Watcher.h0000664000175000017500000000166215161702250017540 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Watcher.h // Baudouin Raoult - ECMWF Jun 98 #ifndef eckit_Watcher_h #define eckit_Watcher_h //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class Watcher { public: // -- Methods virtual ~Watcher() {} virtual void watch() = 0; // -- Class methods static Watcher& dummy(); }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/bases/Loader.cc0000664000175000017500000000161015161702250017500 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/bases/Loader.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- Loader::Loader() : ClassExtent(this) {} Loader::~Loader() {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/os/0000775000175000017500000000000015161702250015311 5ustar alastairalastaireckit-2.0.7/src/eckit/os/AutoAlarm.h0000664000175000017500000000252315161702250017351 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File AutoAlarm.h // Baudouin Raoult - ECMWF Jul 96 #ifndef eckit_AutoAlarm_h #define eckit_AutoAlarm_h //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class AutoAlarm { public: // -- Contructors AutoAlarm(int, bool = false); // -- Destructor ~AutoAlarm(); // - Class methods static bool caught() { return caught_; } private: // No copy allowed AutoAlarm(const AutoAlarm&); AutoAlarm& operator=(const AutoAlarm&); // -- Members using proc = void (*)(int); proc old_; bool saveThrow_; int saveSec_; // -- Class members static bool caught_; static bool throw_; static int sec_; // -- Class methods static void sigAlarm(int); }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/os/SemLocker.cc0000664000175000017500000000565615161702250017520 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/os/SemLocker.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- namespace { struct sembuf _lock[] = { {0, 0, SEM_UNDO}, /* test */ {0, 1, SEM_UNDO}, /* lock */ }; struct sembuf _unlock[] = { {0, -1, SEM_UNDO}, /* ulck */ }; } // namespace SemLocker::SemLocker(int sem, const PathName& path, int maxWaitLock) : sem_(sem), maxWaitLock_(maxWaitLock), path_(path) { int retry = 0; while (retry < maxWaitLock_) { if (semop(sem_, _lock, 2) < 0) { int save = errno; retry++; if (save == EINTR && retry < maxWaitLock_) { continue; } eckit::Log::warning() << "SharedMemoryLoader: Failed to acquire exclusive lock on " << path_ << " " << eckit::Log::syserr << std::endl; // sprintf(message,"ERR: sharedmem:semop:lock(%s)",path); if (retry >= maxWaitLock_) { std::ostringstream os; os << "Failed to acquire semaphore lock for " << path_; throw eckit::FailedSystemCall(os.str()); } eckit::Log::warning() << "Sleeping for " << SLEEP << " seconds" << std::endl; ::sleep(SLEEP); } else { break; } } } SemLocker::~SemLocker() { int retry = 0; while (retry < maxWaitLock_) { if (semop(sem_, _unlock, 1) < 0) { int save = errno; retry++; if (save == EINTR && retry < maxWaitLock_) { continue; } eckit::Log::warning() << "SharedMemoryLoader: Failed to realease exclusive lock on " << path_ << " " << eckit::Log::syserr << std::endl; if (retry >= maxWaitLock_) { std::ostringstream os; os << "Failed to release semaphore lock for " << path_; ASSERT_MSG(retry >= maxWaitLock_, os.str().c_str()); } else { eckit::Log::warning() << "Sleeping for " << SLEEP << " seconds" << std::endl; sleep(SLEEP); } } else { break; } } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/os/SharedInt.cc0000664000175000017500000000350115161702250017500 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/os/SharedInt.h" namespace eckit { SharedInt::SharedInt(const PathName& path, int count) : Semaphore(path, 2 * count) {} SharedInt::~SharedInt() {} void SharedInt::use(int n) { Semaphore::lower(2 * n); } void SharedInt::use(int n, short v) { Semaphore::lower(2 * n, v); } void SharedInt::release(int n) { Semaphore::raise(2 * n); } void SharedInt::release(int n, short v) { Semaphore::raise(2 * n, v); } void SharedInt::newLimit(short val, unsigned short n) { int v; while ((v = semctl(semaphore_, 2 * n + 1, GETVAL, 0)) != val) { if (v < 0) { throw FailedSystemCall("semctl GETVAL"); } short delta = val - v; using U = unsigned short int; struct sembuf set[] = {{ U(2 * n), delta, 0, }, { U(2 * n + 1), delta, 0, }}; SYSCALL(semop(semaphore_, set, NUMBER(set))); } } int SharedInt::limit(int n) const { return Semaphore::get(2 * n + 1); } int SharedInt::free(int n) const { return Semaphore::get(2 * n); } } // namespace eckit eckit-2.0.7/src/eckit/os/BackTrace.h0000664000175000017500000000142215161702250017300 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_os_BackTrace_h #define eckit_os_BackTrace_h #include namespace eckit { //-------------------------------------------------------------------------------------------------- class BackTrace { public: static std::string dump(); }; //-------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/os/Semaphore.h0000664000175000017500000000316215161702250017407 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date May 1996 #ifndef eckit_os_Semaphore_h #define eckit_os_Semaphore_h #include "eckit/filesystem/PathName.h" #include "eckit/thread/Mutex.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class Semaphore { public: // methods Semaphore(const PathName& name, int count = 1); Semaphore(const Semaphore&) = delete; Semaphore& operator=(const Semaphore&) = delete; Semaphore(Semaphore&&) = delete; Semaphore& operator=(Semaphore&&) = delete; ~Semaphore(); void lock(void); void unlock(void); bool tryLock(void); bool test(unsigned short n = 0); int get(int n = 0) const; void set(int, int n = 0); void raise(unsigned short n = 0); void raise(unsigned short n, short v); void lower(unsigned short n = 0); void lower(unsigned short n, short v); pid_t getpid() const; protected: // members int semaphore_; int count_; int level_; Mutex mutex_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/os/Stat.h0000664000175000017500000000236115161702250016377 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_os_Stat_h #define eckit_os_Stat_h #include #include #include #include "eckit/eckit.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// struct to manage differences between stat and stat64 different OS's struct Stat { /// prefer using stat if supports 64 bit using Struct = struct stat; static int stat(const char* path, Struct* s) { return ::stat(path, s); } static int lstat(const char* path, Struct* s) { return ::lstat(path, s); } static int fstat(int fd, Struct* s) { return ::fstat(fd, s); } private: Stat(); ///< non-instantiable }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/os/Password.h0000664000175000017500000000172215161702250017266 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Password.h // Baudouin Raoult - ECMWF Dec 97 #ifndef eckit_Password_h #define eckit_Password_h #include #include "eckit/eckit.h" //----------------------------------------------------------------------------- namespace eckit { //----------------------------------------------------------------------------- class Password { public: static bool check(const std::string&, const std::string&); static std::string salt(const std::string&); }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/os/SignalHandler.cc0000664000175000017500000000377415161702250020346 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/os/SignalHandler.h" #include "eckit/log/Log.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- SignalHandler* SignalHandler::current_ = nullptr; SignalHandler::SignalHandler(void (*handler)(int), SignalHandler::Signal sig) : signal_(sig), next_(current_) { // Log::info() << "Installing signal handler " << signal_ << std::endl; current_ = this; struct sigaction a; a.sa_flags = 0; a.sa_handler = handler; sigemptyset(&a.sa_mask); // volatile int sigtype = sigsetjmp(buf_,1); // if(sigtype == 0) ::sigaction(signal_, &a, &save_); // else { // Log::warning() << "Got signal " << sigtype << std::endl; // throw Abort("Signal received"); //} } SignalHandler::~SignalHandler() { // ASSERT(current_ == this); current_ = next_; struct sigaction ignore; ::sigaction(signal_, &save_, &ignore); // Log::info() << "Removing signal handler " << signal_ << std::endl; } static bool interrupted_ = false; void SignalHandler::interrupt(int sig) { // printf(" >>> received interrupt %d", sig ); if (interrupted_) { ::kill(0, SIGTERM); } // siglongjmp(current_->buf_,sig); interrupted_ = true; } void SignalHandler::checkInterrupt() { // printf(".\n"); if (interrupted_ && !Exception::throwing()) { interrupted_ = false; throw Cancel("Process interrupted"); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/os/System.cc0000664000175000017500000000241715161702250017110 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit.h" #include "eckit/os/System.h" #include "eckit/types/Types.h" #if eckit_HAVE_DLADDR #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include #define ECKIT_ADDRTOPATH_WITH_DLADDR #endif namespace eckit { //---------------------------------------------------------------------------------------------------------------------- std::string System::addrToPath(const void* addr) { std::string result = "/UNKNOWN"; #ifdef ECKIT_ADDRTOPATH_WITH_DLADDR Dl_info info; info.dli_fname = result.c_str(); dladdr(addr, &info); result = info.dli_fname; #endif /// @todo: what do we do with archives (.a) /// * maybe we rely on env variable set? /// * or rely on etc/path return result; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/os/SignalHandler.h0000664000175000017500000000332515161702250020200 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_SignalHandler_h #define eckit_SignalHandler_h #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// @warning This class has not been widely tested, and we don't // know if they delete objects properly when the signal is caugth class SignalHandler { public: // methods enum Signal { SigInt = 2, SigQuit = 3 }; // -- Contructors SignalHandler(void (*)(int) = interrupt, Signal = SigInt); SignalHandler(const SignalHandler&) = delete; SignalHandler& operator=(const SignalHandler&) = delete; SignalHandler(SignalHandler&&) = delete; SignalHandler& operator=(SignalHandler&&) = delete; // -- Destructor ~SignalHandler(); static void checkInterrupt(); private: // methods static void interrupt(int); private: // members int signal_; // unused // sigjmp_buf buf_; struct sigaction save_; SignalHandler* next_; static SignalHandler* current_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/os/Password.cc0000664000175000017500000000337015161702250017425 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/log/Log.h" #include "eckit/os/Password.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- bool Password::check(const std::string& user, const std::string& password) { struct passwd p; struct passwd* dummy; char line[1024]; int n = getpwnam_r(user.c_str(), &p, line, sizeof(line), &dummy); if (n != 0) { Log::error() << "User " << user << " is unknown" << std::endl; return false; } bool match = password == p.pw_passwd; if (match) { Log::error() << "User " << user << " gave an valid password" << std::endl; } else { Log::error() << "User " << user << " gave an invalid password" << std::endl; } return match; } std::string Password::salt(const std::string& user) { struct passwd p; struct passwd* dummy; char line[1024]; int n = getpwnam_r(user.c_str(), &p, line, sizeof(line), &dummy); if (n != 0) { Log::error() << "User " << user << " is unknown" << std::endl; return ""; } char salt[3]; ::strncpy(salt, p.pw_passwd, 2); salt[2] = 0; return salt; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/os/Semaphore.cc0000664000175000017500000001001715161702250017542 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/os/Semaphore.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- struct sembuf _lock[] = { {0, 0, SEM_UNDO}, /* test */ {0, 1, SEM_UNDO}, /* lock */ }; struct sembuf _try_lock[] = { {0, 0, SEM_UNDO | IPC_NOWAIT}, /* test */ {0, 1, SEM_UNDO}, /* lock */ }; struct sembuf _unlock[] = { {0, -1, SEM_UNDO}, /* ulck */ }; Semaphore::Semaphore(const PathName& name, int count) : semaphore_(-1), count_(count), level_(0) { key_t key = ftok(name.localPath(), 1); if (key == key_t(-1) && errno == ENOENT) { name.touch(); key = ftok(name.localPath(), 1); } if (key == key_t(-1)) { throw FailedSystemCall(std::string("ftok(") + name + std::string(")")); } /// @note cannot use Log::debug() of SYSCALL here, because Log may not yet be initialized // std::cout << "Creating semaphore path=" << name << ", count=" << count << ", key=" << hex << // key << dec << std::endl; if ((semaphore_ = semget(key, count_, 0666 | IPC_CREAT)) < 0) { perror("semget failed"), throw FailedSystemCall("semget"); } } Semaphore::~Semaphore() { ASSERT(level_ == 0); } void Semaphore::lock(void) { mutex_.lock(); if (++level_ == 1) { while (semop(semaphore_, _lock, NUMBER(_lock)) < 0) { if (errno != EINTR) { --level_; mutex_.unlock(); throw FailedSystemCall("semop lock"); } } } } bool Semaphore::tryLock(void) { if (!mutex_.tryLock()) { return false; } if (++level_ == 1) { if (semop(semaphore_, _try_lock, NUMBER(_try_lock)) < 0) { --level_; mutex_.unlock(); if (errno == EAGAIN) { return false; } throw FailedSystemCall("semop try_lock"); } } return true; } void Semaphore::unlock(void) { ASSERT(level_ > 0); if (--level_ == 0) { while (semop(semaphore_, _unlock, NUMBER(_unlock)) < 0) { if (errno != EINTR) { ++level_; throw FailedSystemCall("semop unlock"); } } } mutex_.unlock(); } bool Semaphore::test(unsigned short n) { struct sembuf test = {n, 0, IPC_NOWAIT}; if (semop(semaphore_, &test, 1) == 0) { return false; // Free } if (errno == EAGAIN) { return true; // in use } throw FailedSystemCall("semop test"); } pid_t Semaphore::getpid() const { int val; SYSCALL(val = semctl(semaphore_, 0, GETPID)); return val; } void Semaphore::set(int val, int n) { SYSCALL(semctl(semaphore_, n, SETVAL, val)); } int Semaphore::get(int n) const { int val; SYSCALL(val = semctl(semaphore_, n, GETVAL, 0)); return val; } void Semaphore::raise(unsigned short n) { struct sembuf op = {n, 1, SEM_UNDO}; SYSCALL(semop(semaphore_, &op, 1)); } void Semaphore::raise(unsigned short n, short v) { struct sembuf op = {n, v, SEM_UNDO}; SYSCALL(semop(semaphore_, &op, 1)); } void Semaphore::lower(unsigned short n) { struct sembuf op = {n, -1, SEM_UNDO}; SYSCALL(semop(semaphore_, &op, 1)); } void Semaphore::lower(unsigned short n, short v) { short d = -v; struct sembuf op = {n, d, SEM_UNDO}; SYSCALL(semop(semaphore_, &op, 1)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/os/SharedInt.h0000664000175000017500000000205615161702250017346 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Jul 97 #ifndef eckit_SharedInt_h #define eckit_SharedInt_h #include "eckit/os/Semaphore.h" namespace eckit { class SharedInt : public Semaphore { public: SharedInt(const PathName&, int n = 1); ~SharedInt(); void use(int n = 0); void use(int n, short v); void release(int n = 0); void release(int n, short v); int free(int n = 0) const; int limit(int n = 0) const; void newLimit(short val, unsigned short n = 0); private: SharedInt(const SharedInt&) = delete; SharedInt& operator=(const SharedInt&) = delete; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/os/AutoUmask.h0000664000175000017500000000135415161702250017376 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Oct 2016 #ifndef eckit_AutoUmask_h #define eckit_AutoUmask_h #include namespace eckit { class AutoUmask { mode_t umask_; public: explicit AutoUmask(mode_t u = 0) : umask_(::umask(u)) {} ~AutoUmask() { ::umask(umask_); } }; } // namespace eckit #endif eckit-2.0.7/src/eckit/os/System.h0000664000175000017500000000150715161702250016751 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_os_System_h #define eckit_os_System_h #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class System { public: static std::string addrToPath(const void* addr); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/os/AutoAlarm.cc0000664000175000017500000000307715161702250017514 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/os/AutoAlarm.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- bool AutoAlarm::caught_ = false; bool AutoAlarm::throw_ = false; int AutoAlarm::sec_ = 0; void AutoAlarm::sigAlarm(int) { Log::error() << "Alarm signal received" << std::endl; caught_ = true; if (throw_) { throw TimeOut("AutoAlarm", sec_); } } AutoAlarm::AutoAlarm(int sec, bool t) { /// @todo change this to sigaction old_ = ::signal(SIGALRM, sigAlarm); saveThrow_ = throw_; saveSec_ = sec_; throw_ = t; sec_ = sec; caught_ = false; ::alarm(sec); } AutoAlarm::~AutoAlarm() { throw_ = saveThrow_; sec_ = saveSec_; ::signal(SIGALRM, old_); ::alarm(0); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/os/BackTrace.cc0000664000175000017500000000664015161702250017445 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/eckit.h" #if eckit_HAVE_EXECINFO_BACKTRACE || defined(__FreeBSD__) #include // for backtrace #endif #if eckit_HAVE_CXXABI_H #include #endif #include "eckit/exception/Exceptions.h" #include "eckit/os/BackTrace.h" #include "eckit/types/Types.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- std::string BackTrace::dump() { /// @todo implement this using the cxxabi demangle, if CMake detects it std::ostringstream oss; #if (eckit_HAVE_EXECINFO_BACKTRACE || defined(__FreeBSD__)) && !defined(_AIX) static Ordinal count = 0; ++count; #define BS_BUFF_SIZE 256 void* buffer[BS_BUFF_SIZE]; char** strings; int addsize = backtrace(buffer, BS_BUFF_SIZE); oss << "backtrace [" << count << "] stack has " << addsize << " addresses\n"; strings = backtrace_symbols(buffer, addsize); if (strings == nullptr) { oss << " --- no backtrace_symbols found ---\n"; } #if !eckit_HAVE_CXXABI_H for (int s = 0; s < addsize; ++s) oss << strings[s] << std::endl; #else for (int s = 0; s < addsize; ++s) { int status; char buffer[10240]; bool overflow = false; char* p = strings[s]; size_t i = 0; while (*p) { switch (*p) { case ' ': case '(': case ')': case '+': case '\t': oss << *p; if (i) { buffer[i++] = 0; char* d = abi::__cxa_demangle(buffer, nullptr, nullptr, &status); if (status == 0) { oss << d; } else { oss << buffer; } if (d) { free(d); } } i = 0; break; default: if (overflow) { oss << *p; } else { if (i < sizeof(buffer)) { buffer[i++] = *p; } else { overflow = true; for (size_t j = 0; j < i; j++) { oss << buffer[j]; } i = 0; } } } p++; } oss << '\n'; } #endif free(strings); oss << "\nend of backtrace dump ..."; #else oss << "\ndumping backtrace not supported on this system"; #endif return oss.str(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/os/SemLocker.h0000664000175000017500000000242515161702250017351 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Dec 2016 #ifndef eckit_os_SemLocker_h #define eckit_os_SemLocker_h #include "eckit/filesystem/PathName.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- class SemLocker { static const int SLEEP = 1; int sem_; int maxWaitLock_; eckit::PathName path_; public: SemLocker(int sem, const eckit::PathName& path, int maxWaitLock = 60); SemLocker(const SemLocker&) = delete; SemLocker& operator=(const SemLocker&) = delete; SemLocker(SemLocker&&) = delete; SemLocker& operator=(SemLocker&&) = delete; ~SemLocker(); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/mpi/0000775000175000017500000000000015161702250015455 5ustar alastairalastaireckit-2.0.7/src/eckit/mpi/SerialStatus.cc0000664000175000017500000000223715161702250020413 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/mpi/Serial.h" #include "eckit/mpi/SerialStatus.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- SerialStatus::SerialStatus() : source_(Serial::Constants::anySource()), tag_(Serial::Constants::anyTag()), error_(0) {} //---------------------------------------------------------------------------------------------------------------------- void SerialStatus::print(std::ostream& os) const { os << "SerialStatus(" << "source=" << source() << ",tag=" << tag() << ",error=" << error() << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi eckit-2.0.7/src/eckit/mpi/DataType.h0000664000175000017500000000310115161702250017334 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_DataType_h #define eckit_mpi_DataType_h #include #include #include "eckit/mpi/DataType.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class Comm; class Data { public: enum Code { CHAR = 0, WCHAR, SHORT, INT, LONG, SIGNED_CHAR, UNSIGNED_CHAR, UNSIGNED_SHORT, UNSIGNED, UNSIGNED_LONG, FLOAT, DOUBLE, LONG_DOUBLE, // BOOL, COMPLEX, DOUBLE_COMPLEX, // LONG_DOUBLE_COMPLEX, BYTE, PACKED, SHORT_INT, INT_INT, LONG_INT, FLOAT_INT, DOUBLE_INT, LONG_DOUBLE_INT, LONG_LONG, TWO_LONG, TWO_LONG_LONG, MAX_DATA_CODE }; template struct Type { static const char* name(); static Code code(); static size_t size(); }; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/SerialData.h0000664000175000017500000000450615161702250017644 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_SerialData_h #define eckit_mpi_SerialData_h #include #include "eckit/mpi/DataType.h" namespace eckit::mpi { static size_t dataSize[Data::MAX_DATA_CODE] = { /*[Data::CHAR] = */ sizeof(char), /*[Data::WCHAR] = */ sizeof(wchar_t), /*[Data::SHORT] = */ sizeof(short), /*[Data::INT] = */ sizeof(int), /*[Data::LONG] = */ sizeof(long), /*[Data::SIGNED_CHAR] = */ sizeof(signed char), /*[Data::UNSIGNED_CHAR] = */ sizeof(unsigned char), /*[Data::UNSIGNED_SHORT] = */ sizeof(unsigned short), /*[Data::UNSIGNED] = */ sizeof(unsigned int), /*[Data::UNSIGNED_LONG] = */ sizeof(unsigned long), /*[Data::FLOAT] = */ sizeof(float), /*[Data::DOUBLE] = */ sizeof(double), /*[Data::LONG_DOUBLE] = */ sizeof(long double), // /*[Data::BOOL] = */ sizeof(bool), /*[Data::COMPLEX] = */ sizeof(std::complex), /*[Data::DOUBLE_COMPLEX] = */ sizeof(std::complex), // /*[Data::LONG_DOUBLE_COMPLEX] = */ sizeof(std::complex), /*[Data::BYTE] = */ sizeof(char), /*[Data::PACKED] = */ sizeof(char), /*[Data::SHORT_INT] = */ sizeof(std::pair), /*[Data::INT_INT] = */ sizeof(std::pair), /*[Data::LONG_INT] = */ sizeof(std::pair), /*[Data::FLOAT_INT] = */ sizeof(std::pair), /*[Data::DOUBLE_INT] = */ sizeof(std::pair), /*[Data::LONG_DOUBLE_INT] = */ sizeof(std::pair), /*[Data::LONG_LONG] = */ sizeof(long long), /*[Data::TWO_LONG] = */ 2 * sizeof(long), /*[Data::TWO_LONG_LONG] = */ 2 * sizeof(long long), }; } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/ParallelRequest.cc0000664000175000017500000000263115161702250021073 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/CodeLocation.h" #include "eckit/mpi/Parallel.h" #include "eckit/mpi/ParallelRequest.h" namespace eckit { namespace mpi { void MPICall(int code, std::string_view mpifunc, const eckit::CodeLocation& loc); #define MPI_CALL(a) MPICall(a, #a, Here()) //---------------------------------------------------------------------------------------------------------------------- ParallelRequest::ParallelRequest() {} ParallelRequest::ParallelRequest(MPI_Request request) : request_(request) {} void ParallelRequest::print(std::ostream& os) const { os << "ParallelRequest(" << ")"; } int ParallelRequest::request() const { return MPI_Request_c2f(request_); } bool ParallelRequest::test() { int requestCompleted; MPI_CALL(MPI_Test(&request_, &requestCompleted, MPI_STATUS_IGNORE)); return requestCompleted; } //---------------------------------------------------------------------------------------------------------------------- } // namespace mpi } // namespace eckit eckit-2.0.7/src/eckit/mpi/Request.cc0000664000175000017500000000372115161702250017417 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/mpi/Request.h" #include "eckit/mpi/Comm.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class NullRequestContent : public RequestContent { public: virtual ~NullRequestContent() {} virtual void print(std::ostream& os) const { os << "NullRequest()"; } virtual int request() const { return -1; } virtual bool test() { return true; } }; //---------------------------------------------------------------------------------------------------------------------- Request::Request() : content_(new NullRequestContent()) { content_->attach(); } Request::Request(int request) : content_{nullptr} { *this = eckit::mpi::comm().request(request); } Request::Request(RequestContent* p) : content_(p) { content_->attach(); } Request::~Request() { content_->detach(); } Request::Request(const Request& s) : content_(s.content_) { content_->attach(); } Request& Request::operator=(const Request& s) { if (this == &s) { return *this; } if (content_) { content_->detach(); } content_ = s.content_; content_->attach(); return *this; } int Request::request() const { return content_->request(); } bool Request::test() { return content_->test(); } void Request::print(std::ostream& out) const { content_->print(out); } RequestContent::~RequestContent() {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi eckit-2.0.7/src/eckit/mpi/Comm.cc0000664000175000017500000002456215161702250016670 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/mpi/Comm.h" #include #include #include "eckit/eckit_config.h" #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" #include "eckit/utils/Tokenizer.h" namespace eckit::mpi { constexpr bool have_parallel() { #if eckit_HAVE_MPI return true; #else return false; #endif } //---------------------------------------------------------------------------------------------------------------------- class Environment { public: static std::string_view getDefaultCommType() { // Force a given communicator (only required if e.g. running serial applications with MPI) if (const char* forcedComm = ::getenv("ECKIT_MPI_FORCE")) { return forcedComm; // Use parallel communicator if in an MPI environment } if (have_parallel()) { const std::string defaultMPIDetectionVars = "OMPI_COMM_WORLD_SIZE" // OpenMPI ",ALPS_APP_PE" // Cray aprun ",PMI_SIZE" // Intel MPI ",SLURM_STEP_NUM_TASKS"; // slurm srun std::string eckitMPIDetectionVars = eckit::LibResource( "$ECKIT_MPI_DETECTION_VARS;eckitMPIDetectionVars", defaultMPIDetectionVars); std::vector envVars; Tokenizer{','}(eckitMPIDetectionVars, envVars); for (const auto& env : envVars) { if (::getenv(env.c_str())) { return "parallel"; } } } return "serial"; } static Environment& instance() { static Environment env; return env; } void initDefault() { AutoLock lock(mutex_); ASSERT(!default_); Comm* world = CommFactory::build("world", getDefaultCommType()); communicators[world->name()] = world; Comm* self = world->self(); communicators[self->name()] = self; default_ = world; } void setDefault(std::string_view name) { AutoLock lock(mutex_); auto itr = communicators.find(name); if (itr != communicators.end()) { default_ = itr->second; return; } eckit::Log::error() << "Cannot set default communicator to '" << name << "', no communicator with that name was found" << std::endl; eckit::Log::error() << "Current communicators are:" << std::endl; for (itr = communicators.begin(); itr != communicators.end(); ++itr) { eckit::Log::error() << " " << (*itr).first << std::endl; } throw eckit::SeriousBug("No communicator called " + std::string{name}, Here()); } bool hasComm(std::string_view name) { AutoLock lock(mutex_); std::map::iterator itr = communicators.find(name); if (itr != communicators.end()) { return true; } return false; } std::vector listComms() { AutoLock lock(mutex_); std::vector allComms; std::transform(begin(communicators), end(communicators), std::back_inserter(allComms), [](const std::pair& c) { return c.first; }); return allComms; } void finaliseAllComms() { AutoLock lock(mutex_); auto itr = communicators.begin(); for (; itr != communicators.end(); ++itr) { delete itr->second; } communicators.clear(); default_ = nullptr; } Comm& getComm(std::string_view name = {}) { AutoLock lock(mutex_); if (name.empty() && default_) { return *default_; /* most common case first */ } if (!default_) { initDefault(); } if (name.empty()) { ASSERT(default_); /* sanity check init was successful */ return *default_; } auto itr = communicators.find(name); if (itr != communicators.end()) { return *itr->second; } eckit::Log::error() << "No Communicator '" << name << "'" << std::endl; eckit::Log::error() << "Current communicators are:" << std::endl; for (itr = communicators.begin(); itr != communicators.end(); ++itr) { eckit::Log::error() << " " << (*itr).first << std::endl; } throw eckit::SeriousBug("No communicator called " + std::string{name}, Here()); } void addComm(std::string_view name, int comm) { AutoLock lock(mutex_); if (hasComm(name)) { throw SeriousBug("Communicator with name " + std::string{name} + " already exists", Here()); } Comm* pComm = CommFactory::build(name, getDefaultCommType(), comm); communicators.emplace(name, pComm); } void addComm(std::string_view name, Comm* comm) { AutoLock lock(mutex_); if (hasComm(name)) { throw SeriousBug("Communicator with name " + std::string{name} + " already exists", Here()); } communicators.emplace(name, comm); } enum class FreeComm { yes, no }; void unregisterComm(std::string_view name, FreeComm free_comm) { AutoLock lock(mutex_); auto itr = communicators.find(name); if (itr != communicators.end()) { Comm* comm = itr->second; // refuse to unregister the default communicator, world communicator and self communicator if (default_ == comm) { throw SeriousBug("Trying to unregister the default Communicator with name " + std::string{name}, Here()); } if (name == "world") { throw SeriousBug("Trying to unregister the 'world' Communicator", Here()); } if (name == "self") { throw SeriousBug("Trying to unregister the 'self' Communicator", Here()); } if (free_comm == FreeComm::yes) { comm->free(); } communicators.erase(itr); delete comm; } else { throw SeriousBug("Communicator with name " + std::string{name} + " does not exist", Here()); } } Environment() = default; ~Environment() { finaliseAllComms(); } Comm* default_{nullptr}; std::map> communicators; eckit::Mutex mutex_; }; //---------------------------------------------------------------------------------------------------------------------- class CommFactories { public: void registFactory(std::string_view builder, CommFactory* f) { AutoLock lock(mutex_); ASSERT(factories.find(builder) == factories.end()); factories.emplace(builder, f); } void unregistFactory(std::string_view builder) { AutoLock lock(mutex_); factories.erase(std::string{builder}); } CommFactory& getFactory(std::string_view builder) const { AutoLock lock(mutex_); auto j = factories.find(builder); if (j != factories.end()) { return *(j->second); } eckit::Log::error() << "No CommFactory for [" << builder << "]" << std::endl; eckit::Log::error() << "CommFactories are:" << std::endl; for (j = factories.begin(); j != factories.end(); ++j) { eckit::Log::error() << " " << (*j).first << std::endl; } throw eckit::SeriousBug("No CommFactory called " + std::string{builder}, Here()); } static CommFactories& instance() { static CommFactories obj; return obj; } private: CommFactories() {} std::map> factories; mutable eckit::Mutex mutex_; }; CommFactory::CommFactory(std::string_view builder) : builder_(builder) { CommFactories::instance().registFactory(builder, this); } CommFactory::~CommFactory() { CommFactories::instance().unregistFactory(builder_); } Comm* CommFactory::build(std::string_view name, std::string_view builder) { return CommFactories::instance().getFactory(builder).make(name); } Comm* CommFactory::build(std::string_view name, std::string_view builder, int comm) { return CommFactories::instance().getFactory(builder).make(name, comm); } //---------------------------------------------------------------------------------------------------------------------- Comm::Comm(std::string_view name) : name_(name) {} Comm::~Comm() {} //---------------------------------------------------------------------------------------------------------------------- Comm& comm(std::string_view name) { return Environment::instance().getComm(name); } void setCommDefault(std::string_view name) { Environment::instance().setDefault(name); } void addComm(std::string_view name, int comm) { Environment::instance().addComm(name, comm); } void addComm(std::string_view name, Comm* comm) { Environment::instance().addComm(name, comm); } void deleteComm(std::string_view name) { Environment::instance().unregisterComm(name, Environment::FreeComm::yes); } void unregisterComm(std::string_view name) { Environment::instance().unregisterComm(name, Environment::FreeComm::no); } bool hasComm(std::string_view name) { return Environment::instance().hasComm(name); } std::vector listComms() { return Environment::instance().listComms(); } void finaliseAllComms() { return Environment::instance().finaliseAllComms(); } Comm& self() { return comm("self"); } namespace detail { void Assert(int code, const char* msg, const char* file, int line, const char* func) { ::eckit::Assert(code, msg, file, line, func); } } // namespace detail //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi eckit-2.0.7/src/eckit/mpi/Operation.cc0000664000175000017500000000204415161702250017724 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/mpi/Operation.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- Operation::Code sum() { return Operation::SUM; } Operation::Code prod() { return Operation::PROD; } Operation::Code max() { return Operation::MAX; } Operation::Code min() { return Operation::MIN; } Operation::Code maxloc() { return Operation::MAXLOC; } Operation::Code minloc() { return Operation::MINLOC; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi eckit-2.0.7/src/eckit/mpi/Group.h0000664000175000017500000001001015161702250016712 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_Group_h #define eckit_mpi_Group_h #include #include #include #include #include "eckit/memory/Counted.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class GroupContent : public Counted { public: ~GroupContent() override; virtual void print(std::ostream&) const = 0; virtual int group() const = 0; virtual int compare(const GroupContent&) const = 0; virtual GroupContent* difference(const GroupContent&) const = 0; virtual GroupContent* intersection(const GroupContent&) const = 0; virtual GroupContent* union_group(const GroupContent&) const = 0; virtual size_t size() const = 0; virtual int rank() const = 0; virtual GroupContent* excl(const std::vector& ranks) const = 0; virtual GroupContent* incl(const std::vector& ranks) const = 0; virtual GroupContent* range_excl(const std::vector>& ranks) const = 0; virtual GroupContent* range_incl(const std::vector>& ranks) const = 0; virtual std::unordered_map translate_ranks(const std::vector&, const GroupContent&) const = 0; private: friend class Group; }; //---------------------------------------------------------------------------------------------------------------------- /// Group by construction has always a valid content_ /// @invariant content_ is not null class Group { public: // methods /// Null request constructor Group(); /// Group constructor from the Group() integer /// Use only for interfacing with Fortran Group(int); /// Constructor Group(GroupContent*); ~Group(); Group(const Group&); Group& operator=(const Group&); template T& as() { return dynamic_cast(*content_); } template const T& as() const { return dynamic_cast(*content_); } /// Returns this request interpreted as a int by the underlying implementation /// Use only for interfacing with Fortran int group() const; // Compares a group. // // Returns // * an negative integer (< 0): if the groups are not equal (MPI_UNQUAL) // * zero (= 0): if the groups are identical (MPI_IDENT), i.e. members are in exactly the same order in both groups // * a positive integer (> 0): if the groups are similar (MPI_SIMILAR), i.e. both groups contain the same members // but in different order int compare(const Group&) const; Group difference(const Group&) const; Group intersection(const Group&) const; // Can not name the function union because it is a reserved word Group union_group(const Group&) const; size_t size() const; // Returns the rank of the calling process in the given group. int rank() const; Group excl(const std::vector& ranks) const; Group incl(const std::vector& ranks) const; Group range_excl(const std::vector>& ranks) const; Group range_incl(const std::vector>& ranks) const; std::unordered_map translate_ranks(const std::vector&, const Group&) const; private: // methods void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const Group& o) { o.print(s); return s; } private: // members GroupContent* content_; friend class Parallel; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/ParallelGroup.h0000664000175000017500000000476015161702250020406 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_ParallelGroup_h #define eckit_mpi_ParallelGroup_h #define OMPI_SKIP_MPICXX 1 #define MPICH_SKIP_MPICXX 1 #include #include "eckit/mpi/Group.h" namespace eckit { namespace mpi { //---------------------------------------------------------------------------------------------------------------------- class Parallel; class ParallelGroup : public GroupContent { public: ~ParallelGroup() override; private: // constructor ParallelGroup(); ParallelGroup(MPI_Group); private: // methods void print(std::ostream&) const override; int group() const override; int compare(const GroupContent&) const override; int compare(const ParallelGroup&) const; GroupContent* difference(const GroupContent&) const override; ParallelGroup* difference(const ParallelGroup&) const; GroupContent* intersection(const GroupContent&) const override; ParallelGroup* intersection(const ParallelGroup&) const; GroupContent* union_group(const GroupContent&) const override; ParallelGroup* union_group(const ParallelGroup&) const; size_t size() const override; int rank() const override; GroupContent* excl(const std::vector& ranks) const override; GroupContent* incl(const std::vector& ranks) const override; GroupContent* range_excl(const std::vector>& ranks) const override; GroupContent* range_incl(const std::vector>& ranks) const override; std::vector translate_ranks_native(const std::vector& ranks, const ParallelGroup& other) const; std::vector translate_ranks_native(const std::vector& ranks, const GroupContent& other) const; std::unordered_map translate_ranks(const std::vector& ranks, const GroupContent& other) const override; private: // members friend class Parallel; bool valid_; MPI_Group group_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace mpi } // namespace eckit #endif eckit-2.0.7/src/eckit/mpi/Status.h0000664000175000017500000000462515161702250017120 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_Status_h #define eckit_mpi_Status_h #include #include "eckit/memory/Counted.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class StatusContent : public Counted { public: ~StatusContent() override; virtual int source() const = 0; virtual int tag() const = 0; virtual int error() const = 0; virtual void print(std::ostream&) const = 0; }; //---------------------------------------------------------------------------------------------------------------------- class NullStatus : public StatusContent { public: ~NullStatus() override {} int source() const override { return -1; }; int tag() const override { return -1; }; int error() const override { return 1; }; void print(std::ostream&) const override; }; //---------------------------------------------------------------------------------------------------------------------- /// Status by construction has always a valid content_ /// @invariant content_ is not null class Status { public: // methods /// Null Status constructor Status(); /// @pre content is not null Status(StatusContent* content); ~Status(); Status(const Status&); Status& operator=(const Status&); int source() const { return content_->source(); } int tag() const { return content_->tag(); } int error() const { return content_->error(); } template T& as() { return dynamic_cast(*content_); } operator bool() const { return not dynamic_cast(content_); } private: // methods void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const Status& o) { o.print(s); return s; } private: // members StatusContent* content_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/SerialRequest.cc0000664000175000017500000000326615161702250020563 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/mpi/SerialRequest.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/mpi/SerialData.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- SerialRequest::SerialRequest() : request_(-1) {} SerialRequest::~SerialRequest() {} void SerialRequest::print(std::ostream& os) const { os << "SerialRequest(" << "tag->" << tag() << ")"; } int SerialRequest::request() const { return request_; } //---------------------------------------------------------------------------------------------------------------------- SendRequest::SendRequest(const void* buffer, size_t count, Data::Code type, int tag) : buffer_(static_cast(buffer), count * dataSize[type]), count_(count), tag_(tag), type_(type) {} SendRequest::~SendRequest() {} //---------------------------------------------------------------------------------------------------------------------- ReceiveRequest::ReceiveRequest(void* buffer, size_t count, Data::Code type, int tag) : buffer_(buffer), count_(count), tag_(tag), type_(type) {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi eckit-2.0.7/src/eckit/mpi/SerialRequest.h0000664000175000017500000000532315161702250020421 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_SerialRequest_h #define eckit_mpi_SerialRequest_h #include "eckit/io/Buffer.h" #include "eckit/mpi/DataType.h" #include "eckit/mpi/Request.h" namespace eckit::mpi { class SerialRequestPool; class Serial; //---------------------------------------------------------------------------------------------------------------------- class SerialRequest : public RequestContent { public: // methods SerialRequest(); ~SerialRequest() override; int request() const override; virtual int tag() const = 0; virtual bool isReceive() const = 0; bool test() override { return true; } private: // methods void print(std::ostream&) const override; bool handled() const { return handled_; } void handled(bool v) { handled_ = v; } private: // members friend class SerialRequestPool; friend class Serial; int request_; bool handled_{false}; }; //---------------------------------------------------------------------------------------------------------------------- class SendRequest : public SerialRequest { public: // methods SendRequest(const void* buffer, size_t count, Data::Code type, int tag); ~SendRequest() override; bool isReceive() const override { return false; } const void* buffer() const { return buffer_; } size_t count() const { return count_; } int tag() const override { return tag_; } Data::Code type() const { return type_; } private: eckit::Buffer buffer_; size_t count_; int tag_; Data::Code type_; }; //---------------------------------------------------------------------------------------------------------------------- class ReceiveRequest : public SerialRequest { public: // methods ReceiveRequest(void* buffer, size_t count, Data::Code type, int tag); virtual bool isReceive() const { return true; } /// Non const access is needed void* buffer() { return buffer_; } const void* buffer() const { return buffer_; } size_t count() const { return count_; } virtual int tag() const { return tag_; } Data::Code type() const { return type_; } private: void* buffer_; size_t count_; int tag_; Data::Code type_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/Parallel.cc0000664000175000017500000006363015161702250017530 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/mpi/Parallel.h" #include #include #include #include #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" #include "eckit/mpi/ParallelGroup.h" #include "eckit/mpi/ParallelRequest.h" #include "eckit/mpi/ParallelStatus.h" #include "eckit/runtime/Main.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" namespace eckit { namespace mpi { //---------------------------------------------------------------------------------------------------------------------- std::atomic initCounter{}; // zero-initialized //---------------------------------------------------------------------------------------------------------------------- class MPIError : public eckit::Exception { public: MPIError(const std::string& msg, const eckit::CodeLocation& loc) : eckit::Exception(msg, loc) { std::ostringstream s; s << "MPI Error: " << msg << " in " << loc; reason(s.str()); } }; //---------------------------------------------------------------------------------------------------------------------- void MPICall(int code, std::string_view mpifunc, const eckit::CodeLocation& loc) { if (code != MPI_SUCCESS) { char error[10240]; int len = sizeof(error) - 1; MPI_Error_string(code, error, &len); error[len] = 0; std::ostringstream oss; oss << "MPI call failed with error '" << error << "' while calling " << mpifunc; throw MPIError(oss.str(), loc); } } #define MPI_CALL(a) MPICall(a, #a, Here()) //---------------------------------------------------------------------------------------------------------------------- namespace { static MPI_Datatype PARALLEL_TWO_LONG() { static MPI_Datatype mpi_datatype = [] { MPI_Datatype mpi_datatype; MPI_Type_contiguous(2, MPI_LONG, &mpi_datatype); MPI_Type_commit(&mpi_datatype); return mpi_datatype; }(); return mpi_datatype; } static MPI_Datatype PARALLEL_TWO_LONG_LONG() { static MPI_Datatype mpi_datatype = [] { MPI_Datatype mpi_datatype; MPI_Type_contiguous(2, MPI_LONG_LONG, &mpi_datatype); MPI_Type_commit(&mpi_datatype); return mpi_datatype; }(); return mpi_datatype; } struct LocalMutex { operator eckit::Mutex&() { return mutex_; } static LocalMutex& instance() { static LocalMutex instance; return instance; } private: LocalMutex() = default; eckit::Mutex mutex_; }; struct WorkaroundSegvAtExit { // This is a workaround for when a SIGSEGV is happening during the // program teardown phase after the main() scope ends. eckit::mpi is designed to // automatically call MPI_Finalize at this point when needed. // Some MPI implementations trigger a SIGSEGV. This is e.g. the case with hpcx-openmpi/2.14.0 // with CUDA-aware features enabled. // // To enable the workaround, please export `ECKIT_MPI_WORKAROUND_SEGV_AT_EXIT=1` // // For more details see https://github.com/ecmwf/eckit/pull/171 static void enabled(bool value) { instance().enabled_ = value; } static bool enabled() { return instance().enabled_; } static void MPI_Finalize() { // This function calls MPI_Finalize, traps a SIGSEGV when it happens, ignores it, // and restores the SIGSEGV signal handler to the default, to handle further SIGSEGV. static std::jmp_buf sigsegv_occured; std::signal(SIGSEGV, [](int) { std::longjmp(sigsegv_occured, true); }); if (setjmp(sigsegv_occured)) { std::signal(SIGSEGV, SIG_DFL); // std::cerr << "WARNING: Trapped SIGSEGV fault during MPI_Finalize() after main()" << std::endl; return; } MPI_CALL(::MPI_Finalize()); } private: static WorkaroundSegvAtExit& instance() { static WorkaroundSegvAtExit instance; return instance; } bool enabled_{false}; }; } // namespace // define MPI_LONG_LONG if not existing to avoid compilation errors #ifndef MPI_LONG_LONG #define MPI_LONG_LONG MPI_LONG #endif static MPI_Datatype toType(Data::Code code) { static MPI_Datatype mpi_datatype_[Data::MAX_DATA_CODE] = { /*[Data::CHAR] = */ MPI_CHAR, /*[Data::WCHAR] = */ MPI_WCHAR, /*[Data::SHORT] = */ MPI_SHORT, /*[Data::INT] = */ MPI_INT, /*[Data::LONG] = */ MPI_LONG, /*[Data::SIGNED_CHAR] = */ MPI_SIGNED_CHAR, /*[Data::SIGNED_CHAR] = */ MPI_UNSIGNED_CHAR, /*[Data::UNSIGNED_SHORT] = */ MPI_UNSIGNED_SHORT, /*[Data::UNSIGNED] = */ MPI_UNSIGNED, /*[Data::UNSIGNED_LONG] = */ MPI_UNSIGNED_LONG, /*[Data::FLOAT] = */ MPI_FLOAT, /*[Data::DOUBLE] = */ MPI_DOUBLE, /*[Data::LONG_DOUBLE] = */ MPI_LONG_DOUBLE, // /*[Data::BOOL] = */ MPI_BOOL, /*[Data::COMPLEX] = */ MPI_COMPLEX, /*[Data::DOUBLE_COMPLEX] = */ MPI_DOUBLE_COMPLEX, // /*[Data::LONG_DOUBLE_COMPLEX] = */ MPI_LONG_DOUBLE_COMPLEX, /*[Data::BYTE] = */ MPI_BYTE, /*[Data::PACKED] = */ MPI_PACKED, /*[Data::SHORT_INT] = */ MPI_SHORT_INT, /*[Data::INT_INT] = */ MPI_2INT, /*[Data::LONG_INT] = */ MPI_LONG_INT, /*[Data::FLOAT_INT] = */ MPI_FLOAT_INT, /*[Data::DOUBLE_INT] = */ MPI_DOUBLE_INT, /*[Data::LONG_DOUBLE_INT] = */ MPI_LONG_DOUBLE_INT, /*[Data::LONG_LONG] = */ MPI_LONG_LONG, /*[Data::TWO_LONG] = */ PARALLEL_TWO_LONG(), /*[Data::TWO_LONG_LONG] = */ PARALLEL_TWO_LONG_LONG(), }; return mpi_datatype_[code]; } //---------------------------------------------------------------------------------------------------------------------- static MPI_Op mpi_opcode[Operation::MAX_OPERATION_CODE] = { /*[Data::SUM] = */ MPI_SUM, /*[Data::PROD] = */ MPI_PROD, /*[Data::MAX] = */ MPI_MAX, /*[Data::MIN] = */ MPI_MIN, /*[Data::MAXLOC] = */ MPI_MAXLOC, /*[Data::MINLOC] = */ MPI_MINLOC}; static MPI_Op toOp(Operation::Code code) { return mpi_opcode[code]; } //---------------------------------------------------------------------------------------------------------------------- namespace { size_t getRank(MPI_Comm comm) { int rank; MPI_CALL(MPI_Comm_rank(comm, &rank)); return size_t(rank); } size_t getSize(MPI_Comm comm) { int size; MPI_CALL(MPI_Comm_size(comm, &size)); return size_t(size); } } // namespace //---------------------------------------------------------------------------------------------------------------------- Parallel::Parallel(std::string_view name) : Comm(name) /* don't use member initialisation list */ { if (initCounter == 0) { initialize(); } initCounter++; comm_ = MPI_COMM_WORLD; rank_ = getRank(comm_); size_ = getSize(comm_); } Parallel::Parallel(std::string_view name, MPI_Comm comm, bool) : Comm(name) /* don't use member initialisation list */ { if (initCounter == 0) { initialize(); } initCounter++; comm_ = comm; rank_ = getRank(comm_); size_ = getSize(comm_); } Parallel::Parallel(std::string_view name, int comm) : Comm(name) { initCounter++; comm_ = MPI_Comm_f2c(comm); rank_ = getRank(comm_); size_ = getSize(comm_); } Parallel::~Parallel() { initCounter--; if (initCounter == 0) { finalize(); } } Comm* Parallel::self() const { return new Parallel("self", MPI_COMM_SELF, true); } void Parallel::initialize() { eckit::AutoLock lock(LocalMutex::instance()); if (!initialized()) { int argc(0); char** argv = nullptr; if (eckit::Main::ready()) { argc = eckit::Main::instance().argc(); argv = eckit::Main::instance().argv(); } std::string MPIInitThread = eckit::Resource("MPIInitThread;$ECKIT_MPI_INIT_THREAD", "NONE"); WorkaroundSegvAtExit::enabled(eckit::Resource("$ECKIT_MPI_WORKAROUND_SEGV_AT_EXIT", false)); if (MPIInitThread == "NONE") { MPI_CALL(MPI_Init(&argc, &argv)); } else { int required; if (MPIInitThread == "MPI_THREAD_SINGLE") required = MPI_THREAD_SINGLE; //< only one thread executes else if (MPIInitThread == "MPI_THREAD_FUNNELED") required = MPI_THREAD_FUNNELED; //< process may be multi-threaded, but only the main thread will make // MPI calls else if (MPIInitThread == "MPI_THREAD_SERIALIZED") required = MPI_THREAD_SERIALIZED; //< the process may be multi-threaded, and multiple threads may make // MPI calls, but only one at a time else if (MPIInitThread == "MPI_THREAD_MULTIPLE") required = MPI_THREAD_MULTIPLE; //< multiple threads may call MPI, with no restrictions else { std::ostringstream msg; msg << "Value of mpiInitThread is unrecognised: " << MPIInitThread << " -- Valid values are: NONE, MPI_THREAD_SINGLE, MPI_THREAD_FUNNELED, MPI_THREAD_SERIALIZED and " "MPI_THREAD_MULTIPLE"; throw eckit::UserError(msg.str(), Here()); } int provided; MPI_CALL(MPI_Init_thread(&argc, &argv, required, &provided)); if (provided != required) { std::ostringstream msg; msg << "MPI_Init_thread provides different thread support than requested -" << " requested: " << MPIInitThread; if (provided == MPI_THREAD_SINGLE) msg << " provided: " << "MPI_THREAD_SINGLE"; else if (provided == MPI_THREAD_FUNNELED) msg << " provided: " << "MPI_THREAD_FUNNELED"; else if (provided == MPI_THREAD_SERIALIZED) msg << " provided: " << "MPI_THREAD_SERIALIZED"; else if (provided == MPI_THREAD_MULTIPLE) msg << " provided: " << "MPI_THREAD_MULTIPLE"; else msg << " provided: " << " UNKNOWN (" << provided << ")"; throw eckit::UserError(msg.str(), Here()); } } } } void Parallel::finalize() { if (not finalized()) { if (WorkaroundSegvAtExit::enabled()) { WorkaroundSegvAtExit::MPI_Finalize(); } else { MPI_CALL(MPI_Finalize()); } } } bool Parallel::initialized() { int result = 1; MPI_CALL(MPI_Initialized(&result)); return bool(result); } bool Parallel::finalized() { int result = 1; MPI_CALL(MPI_Finalized(&result)); return bool(result); } std::string Parallel::processorName() const { char hostname[256]; int size = sizeof(hostname); MPI_CALL(MPI_Get_processor_name(hostname, &size)); return hostname; } size_t Parallel::remoteSize() const { int s; MPI_CALL(MPI_Comm_remote_size(comm_, &s)); return ((size_t)s); } void Parallel::barrier() const { MPI_CALL(MPI_Barrier(comm_)); } Request Parallel::iBarrier() const { Request req(new ParallelRequest()); MPI_CALL(MPI_Ibarrier(comm_, toRequest(req))); return req; } void Parallel::abort(int errorcode) const { MPI_CALL(MPI_Abort(comm_, errorcode)); } Status Parallel::wait(Request& req) const { Status st = createStatus(); MPI_CALL(MPI_Wait(toRequest(req), toStatus(st))); return st; } std::vector Parallel::waitAll(std::vector& req) const { int count = req.size(); std::vector st(count); std::vector st_(count); std::vector req_(count); for (int i = 0; i < count; i++) { st[i] = createStatus(); st_[i] = *(toStatus(st[i])); req_[i] = *(toRequest(req[i])); } MPI_CALL(MPI_Waitall(count, &req_[0], &st_[0])); for (int i = 0; i < count; i++) { *(toStatus(st[i])) = st_[i]; *(toRequest(req[i])) = req_[i]; } return st; } Status Parallel::waitAny(std::vector& req, int& ireq) const { int count = req.size(); Status st = createStatus(); std::vector req_(count); for (int i = 0; i < count; i++) { req_[i] = *(toRequest(req[i])); } MPI_CALL(MPI_Waitany(count, &req_[0], &ireq, toStatus(st))); for (int i = 0; i < count; i++) { *(toRequest(req[i])) = req_[i]; } return st; } Status Parallel::probe(int source, int tag) const { Status st = createStatus(); MPI_CALL(MPI_Probe(source, tag, comm_, toStatus(st))); return st; } Status Parallel::iProbe(int source, int tag) const { Status st = createStatus(); int flag = 0; MPI_CALL(MPI_Iprobe(source, tag, comm_, &flag, toStatus(st))); return flag ? st : Status{}; } int Parallel::anySource() const { return MPI_ANY_SOURCE; } int Parallel::anyTag() const { return MPI_ANY_TAG; } int Parallel::undefined() const { return MPI_UNDEFINED; } int Parallel::procNull() const { return MPI_PROC_NULL; } size_t Parallel::getCount(Status& st, Data::Code type) const { int count = 0; MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Get_count(toStatus(st), mpitype, &count)); ASSERT(count >= 0); return size_t(count); } void Parallel::broadcast(void* buffer, size_t count, Data::Code type, size_t root) const { ASSERT(root < size_t(std::numeric_limits::max())); ASSERT(count < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Bcast(buffer, int(count), mpitype, int(root), comm_)); } void Parallel::gather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type, size_t root) const { ASSERT(sendcount < size_t(std::numeric_limits::max())); ASSERT(recvcount < size_t(std::numeric_limits::max())); ASSERT(root < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Gather(const_cast(sendbuf), int(sendcount), mpitype, recvbuf, int(recvcount), mpitype, int(root), comm_)); } void Parallel::scatter(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type, size_t root) const { ASSERT(sendcount < size_t(std::numeric_limits::max())); ASSERT(recvcount < size_t(std::numeric_limits::max())); ASSERT(root < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Scatter(const_cast(sendbuf), int(sendcount), mpitype, recvbuf, int(recvcount), mpitype, int(root), comm_)); } void Parallel::gatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int recvcounts[], const int displs[], Data::Code type, size_t root) const { ASSERT(sendcount < size_t(std::numeric_limits::max())); ASSERT(root < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Gatherv(const_cast(sendbuf), int(sendcount), mpitype, recvbuf, const_cast(recvcounts), const_cast(displs), mpitype, int(root), comm_)); } void Parallel::scatterv(const void* sendbuf, const int sendcounts[], const int displs[], void* recvbuf, size_t recvcount, Data::Code type, size_t root) const { ASSERT(recvcount < size_t(std::numeric_limits::max())); ASSERT(root < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Scatterv(const_cast(sendbuf), const_cast(sendcounts), const_cast(displs), mpitype, recvbuf, int(recvcount), mpitype, int(root), comm_)); } void Parallel::reduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code type, Operation::Code op, size_t root) const { ASSERT(count < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_Op mpiop = toOp(op); MPI_CALL(MPI_Reduce(const_cast(sendbuf), recvbuf, int(count), mpitype, mpiop, int(root), comm_)); } void Parallel::reduceInPlace(void* sendrecvbuf, size_t count, Data::Code type, Operation::Code op, size_t root) const { ASSERT(count < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_Op mpiop = toOp(op); if (rank() == root) { MPI_CALL(MPI_Reduce(MPI_IN_PLACE, sendrecvbuf, int(count), mpitype, mpiop, int(root), comm_)); } else { MPI_CALL(MPI_Reduce(sendrecvbuf, sendrecvbuf, int(count), mpitype, mpiop, int(root), comm_)); } } void Parallel::allReduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code type, Operation::Code op) const { ASSERT(count < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_Op mpiop = toOp(op); MPI_CALL(MPI_Allreduce(const_cast(sendbuf), recvbuf, int(count), mpitype, mpiop, comm_)); } void Parallel::allReduceInPlace(void* sendrecvbuf, size_t count, Data::Code type, Operation::Code op) const { ASSERT(count < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_Op mpiop = toOp(op); MPI_CALL(MPI_Allreduce(MPI_IN_PLACE, sendrecvbuf, int(count), mpitype, mpiop, comm_)); } void Parallel::allGather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type) const { ASSERT(sendcount < size_t(std::numeric_limits::max())); ASSERT(recvcount < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL( MPI_Allgather(const_cast(sendbuf), int(sendcount), mpitype, recvbuf, int(recvcount), mpitype, comm_)); } void Parallel::allGatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int recvcounts[], const int displs[], Data::Code type) const { ASSERT(sendcount < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Allgatherv(const_cast(sendbuf), int(sendcount), mpitype, recvbuf, const_cast(recvcounts), const_cast(displs), mpitype, comm_)); } void Parallel::allToAll(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type) const { ASSERT(sendcount < size_t(std::numeric_limits::max())); ASSERT(recvcount < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL( MPI_Alltoall(const_cast(sendbuf), int(sendcount), mpitype, recvbuf, int(recvcount), mpitype, comm_)); } void Parallel::allToAllv(const void* sendbuf, const int sendcounts[], const int sdispls[], void* recvbuf, const int recvcounts[], const int rdispls[], Data::Code type) const { MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Alltoallv(const_cast(sendbuf), const_cast(sendcounts), const_cast(sdispls), mpitype, recvbuf, const_cast(recvcounts), const_cast(rdispls), mpitype, comm_)); } Status Parallel::receive(void* recv, size_t count, Data::Code type, int source, int tag) const { ASSERT(count < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); Status status = createStatus(); MPI_CALL(MPI_Recv(recv, int(count), mpitype, source, tag, comm_, toStatus(status))); return status; } void Parallel::send(const void* send, size_t count, Data::Code type, int dest, int tag) const { ASSERT(count < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Send(const_cast(send), int(count), mpitype, dest, tag, comm_)); } void Parallel::synchronisedSend(const void* send, size_t count, Data::Code type, int dest, int tag) const { ASSERT(count < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Ssend(const_cast(send), int(count), mpitype, dest, tag, comm_)); } Request Parallel::iReceive(void* recv, size_t count, Data::Code type, int source, int tag) const { ASSERT(count < size_t(std::numeric_limits::max())); Request req(new ParallelRequest()); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Irecv(recv, int(count), mpitype, source, tag, comm_, toRequest(req))); return req; } Request Parallel::iSend(const void* send, size_t count, Data::Code type, int dest, int tag) const { ASSERT(count < size_t(std::numeric_limits::max())); Request req(new ParallelRequest()); MPI_Datatype mpitype = toType(type); MPI_CALL(MPI_Isend(const_cast(send), int(count), mpitype, dest, tag, comm_, toRequest(req))); return req; } Status Parallel::sendReceiveReplace(void* sendrecv, size_t count, Data::Code type, int dest, int sendtag, int source, int recvtag) const { ASSERT(count < size_t(std::numeric_limits::max())); MPI_Datatype mpitype = toType(type); Status status = createStatus(); MPI_CALL( MPI_Sendrecv_replace(sendrecv, int(count), mpitype, dest, sendtag, source, recvtag, comm_, toStatus(status))); return status; } Comm& Parallel::split(int color, const std::string& name) const { if (hasComm(name.c_str())) { throw SeriousBug("Communicator with name " + name + " already exists"); } MPI_Comm new_mpi_comm; MPI_CALL(MPI_Comm_split(comm_, color, rank(), &new_mpi_comm)); Comm* newcomm = new Parallel(name, new_mpi_comm, true); addComm(name.c_str(), newcomm); return *newcomm; } void Parallel::free() { if (comm_ == MPI_COMM_WORLD) { throw SeriousBug("Cannot free MPI_COMM_WORLD communicator"); } if (comm_ == MPI_COMM_SELF) { throw SeriousBug("Cannot free MPI_COMM_SELF communicator"); } MPI_CALL(MPI_Comm_free(&comm_)); rank_ = 0; size_ = 0; } void Parallel::print(std::ostream& os) const { os << "Parallel()"; /// @note maybe add information about the MPI backend: opem-mpi? mpich? etc... } MPI_Status* Parallel::toStatus(Status& st) { return &(st.as().status_); } Status Parallel::createStatus() { return Status(new ParallelStatus()); } Request Parallel::request(int request) const { return Request(new ParallelRequest(MPI_Request_f2c(request))); } Group Parallel::group(int group) const { return Group(new ParallelGroup(MPI_Group_f2c(group))); } Group Parallel::group() const { MPI_Group g; MPI_CALL(MPI_Comm_group(comm_, &g)); return Group(new ParallelGroup(g)); } Group Parallel::remoteGroup() const { MPI_Group g; MPI_CALL(MPI_Comm_remote_group(comm_, &g)); return Group(new ParallelGroup(g)); } Comm& Parallel::create(const Group& group, const std::string& name) const { MPI_Comm new_mpi_comm; MPI_CALL(MPI_Comm_create(comm_, dynamic_cast(*group.content_).group_, &new_mpi_comm)); Comm* newcomm = new Parallel(name, new_mpi_comm, true); addComm(name.c_str(), newcomm); return *newcomm; }; Comm& Parallel::create(const Group& group, int tag, const std::string& name) const { MPI_Comm new_mpi_comm; MPI_CALL( MPI_Comm_create_group(comm_, dynamic_cast(*group.content_).group_, tag, &new_mpi_comm)); Comm* newcomm = new Parallel(name, new_mpi_comm, true); addComm(name.c_str(), newcomm); return *newcomm; }; MPI_Request* Parallel::toRequest(Request& req) { return &(req.as().request_); } MPI_Comm Parallel::MPIComm() const { return comm_; } int Parallel::communicator() const { return MPI_Comm_c2f(comm_); } eckit::SharedBuffer Parallel::broadcastFile(const PathName& filepath, size_t root) const { ASSERT(root < size()); bool isRoot = rank() == root; eckit::CountedBuffer* buffer = nullptr; struct BFileOp { int err_; size_t len_; } op = {0, 0}; errno = 0; if (isRoot) { try { std::unique_ptr dh(filepath.fileHandle()); op.len_ = dh->openForRead(); AutoClose closer(*dh); buffer = new eckit::CountedBuffer(op.len_); dh->read(buffer->data(), op.len_); if (filepath.isDir()) { op.err_ = EISDIR; } } catch (Exception&) { op.err_ = errno; } } broadcast(&op, sizeof(op), Data::BYTE, root); errno = op.err_; // set errno to ensure consistent error messages across MPI tasks if (op.err_) { throw CantOpenFile(filepath); } if (not op.len_) { throw ShortFile(filepath); } if (!isRoot) { buffer = new eckit::CountedBuffer(op.len_); } broadcast(*buffer, op.len_, Data::BYTE, root); return eckit::SharedBuffer(buffer); } static CommBuilder ParallelBuilder("parallel"); //---------------------------------------------------------------------------------------------------------------------- } // namespace mpi } // namespace eckit eckit-2.0.7/src/eckit/mpi/ParallelStatus.h0000664000175000017500000000243415161702250020571 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_ParallelStatus_h #define eckit_mpi_ParallelStatus_h #define OMPI_SKIP_MPICXX 1 #define MPICH_SKIP_MPICXX 1 #include #include #include "eckit/mpi/Status.h" namespace eckit { namespace mpi { //---------------------------------------------------------------------------------------------------------------------- class Parallel; class ParallelStatus : public StatusContent { private: // methods int source() const override { return status_.MPI_SOURCE; } int tag() const override { return status_.MPI_TAG; } int error() const override { return status_.MPI_ERROR; } void print(std::ostream&) const override; private: // members friend class Parallel; MPI_Status status_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace mpi } // namespace eckit #endif eckit-2.0.7/src/eckit/mpi/ParallelRequest.h0000664000175000017500000000235615161702250020741 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_ParallelRequest_h #define eckit_mpi_ParallelRequest_h #define OMPI_SKIP_MPICXX 1 #define MPICH_SKIP_MPICXX 1 #include #include "eckit/mpi/Request.h" namespace eckit { namespace mpi { //---------------------------------------------------------------------------------------------------------------------- class Parallel; class ParallelRequest : public RequestContent { private: // constructor ParallelRequest(); ParallelRequest(MPI_Request); private: // methods void print(std::ostream&) const override; int request() const override; bool test() override; private: // members friend class Parallel; MPI_Request request_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace mpi } // namespace eckit #endif eckit-2.0.7/src/eckit/mpi/Serial.cc0000664000175000017500000003557415161702250017221 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/mpi/Serial.h" #include #include #include #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" #include "eckit/maths/Functions.h" #include "eckit/mpi/Group.h" #include "eckit/mpi/SerialData.h" #include "eckit/mpi/SerialRequest.h" #include "eckit/mpi/SerialStatus.h" #include "eckit/runtime/Main.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class SerialRequestPool { public: static SerialRequestPool& instance() { static SerialRequestPool request_pool; return request_pool; } SerialRequestPool(const SerialRequestPool&) = delete; SerialRequestPool& operator=(const SerialRequestPool&) = delete; SerialRequestPool(SerialRequestPool&&) = delete; SerialRequestPool& operator=(SerialRequestPool&&) = delete; Request createSendRequest(const void* buffer, size_t count, Data::Code type, int tag) { Request r = registerRequest(new SendRequest(buffer, count, type, tag)); send_[tag].push_back(r); return r; } Request createReceiveRequest(void* buffer, size_t count, Data::Code type, int tag) { SerialRequest* request = new ReceiveRequest(buffer, count, type, tag); return registerRequest(request); } Request operator[](int request) { return requests_[request]; } SendRequest& matchingSendRequest(const ReceiveRequest& req) { return matchingSendRequest(req.tag()); } SendRequest& matchingSendRequest(int tag) { if (tag == anyTag()) { std::map >::iterator it = send_.begin(); for (; it != send_.end(); ++it) { std::deque& sends = it->second; if (sends.size()) { Request send = sends.front(); sends.pop_front(); return send.as(); } } throw eckit::Exception("No send requests available", Here()); } ASSERT(send_.count(tag) > 0); ASSERT(send_[tag].size()); Request send = send_[tag].front(); send_[tag].pop_front(); return send.as(); } SendRequest* matchNextSendRequest(int tag) { Request send; if (tag == anyTag()) { for (const auto& item : send_) { const auto& sends = item.second; if (sends.size()) { send = sends.front(); return &send.as(); } } return nullptr; } ASSERT(send_.count(tag) > 0); ASSERT(send_[tag].size()); if (send_.find(tag) != end(send_)) { send = send_.at(tag).front(); return &send.as(); } return nullptr; } void lock() { mutex_.lock(); } void unlock() { mutex_.unlock(); } static constexpr int anyTag() { return Serial::Constants::anyTag(); } private: Request registerRequest(SerialRequest* request) { ++n_; if (size_t(n_) == requests_.size()) { n_ = 0; } request->request_ = n_; Request r(request); requests_[n_] = r; return r; } SerialRequestPool() { n_ = -1; requests_.resize(100); } ~SerialRequestPool() {} std::vector requests_; std::map > send_; int n_; eckit::Mutex mutex_; ///< instance() creation is thread safe, but access thereon isn't so we need a mutex }; //---------------------------------------------------------------------------------------------------------------------- Serial::Serial(std::string_view name) : Comm(name) { rank_ = 0; size_ = 1; } Serial::Serial(std::string_view name, int) : Comm(name) { rank_ = 0; size_ = 1; } Serial::~Serial() {} Comm* Serial::self() const { return new Serial("self"); } std::string Serial::processorName() const { return Main::hostname(); } size_t Serial::remoteSize() const { return 0; }; void Serial::barrier() const { return; } Request Serial::iBarrier() const { return Request(); } Comm& Serial::split(int /*color*/, const std::string& name) const { if (hasComm(name.c_str())) { throw SeriousBug("Communicator with name " + name + " already exists"); } Comm* newcomm = new Serial(name); addComm(name.c_str(), newcomm); return *newcomm; } void Serial::free() { // nothing todo } void Serial::abort(int) const { // Don't use std::abort as it would raise SIGABRT. // MPI_Abort also does not raise SIGABRT std::exit(EXIT_FAILURE); } Status Serial::wait(Request& req) const { AutoLock lock(SerialRequestPool::instance()); auto& serialRequest = req.as(); // Return early if request was already handled. if (serialRequest.handled()) { return new SerialStatus{}; } // Continue if request was not yet handled. serialRequest.handled(true); // Only do memcpy when waiting for a ReceiveRequest, and return status. if (req.as().isReceive()) { ReceiveRequest& recvReq = req.as(); SendRequest& sendReq = SerialRequestPool::instance().matchingSendRequest(recvReq); if (sendReq.count() > 0) { ::memcpy(recvReq.buffer(), sendReq.buffer(), sendReq.count() * dataSize[sendReq.type()]); } SerialStatus* st = new SerialStatus(); (*st).count_ = sendReq.count(); (*st).source_ = 0; (*st).tag_ = sendReq.tag(); (*st).error_ = 0; return Status(st); } // For SendRequests, do nothing and return a default SerialStatus return new SerialStatus{}; } std::vector Serial::waitAll(std::vector& requests) const { std::vector statuses; statuses.reserve(requests.size()); for (auto& req : requests) { statuses.push_back(wait(req)); } return statuses; } Status Serial::waitAny(std::vector& requests, int& index) const { for (size_t i = 0; i < requests.size(); ++i) { if (!requests[i].as().handled()) { Status status = wait(requests[i]); index = i; return status; } } index = undefined(); return new SerialStatus{}; } Status Serial::probe(int source, int tag) const { ASSERT(source == 0 || source == Serial::Constants::anySource()); SerialStatus* st = new SerialStatus(); SendRequest* req = nullptr; while (not req) { req = SerialRequestPool::instance().matchNextSendRequest(tag); } (*st).count_ = req->count(); (*st).tag_ = req->tag(); (*st).source_ = 0; (*st).error_ = 0; return Status(st); } Status Serial::iProbe(int source, int tag) const { ASSERT(source == 0 || source == Serial::Constants::anySource()); SendRequest* req = SerialRequestPool::instance().matchNextSendRequest(tag); if (not req) { return Status{}; // Null status } SerialStatus* st = new SerialStatus{}; (*st).count_ = req->count(); (*st).tag_ = req->tag(); (*st).source_ = 0; (*st).error_ = 0; return Status(st); } int Serial::anySource() const { return Serial::Constants::anySource(); } int Serial::anyTag() const { return Serial::Constants::anyTag(); } int Serial::undefined() const { return Serial::Constants::undefined(); } int Serial::procNull() const { return Serial::Constants::procNull(); } size_t Serial::getCount(Status& st, Data::Code) const { return st.as().count_; } void Serial::broadcast(void*, size_t, Data::Code, size_t) const { return; } void Serial::gather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t, Data::Code type, size_t) const { if (recvbuf != sendbuf && sendcount > 0) { memcpy(recvbuf, sendbuf, sendcount * dataSize[type]); } } void Serial::scatter(const void* sendbuf, size_t, void* recvbuf, size_t recvcount, Data::Code type, size_t) const { if (recvbuf != sendbuf && recvcount > 0) { memcpy(recvbuf, sendbuf, recvcount * dataSize[type]); } } void Serial::gatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int[], const int[], Data::Code type, size_t) const { if (recvbuf != sendbuf && sendcount > 0) { memcpy(recvbuf, sendbuf, sendcount * dataSize[type]); } } void Serial::scatterv(const void* sendbuf, const int[], const int[], void* recvbuf, size_t recvcount, Data::Code type, size_t) const { if (recvbuf != sendbuf && recvcount > 0) { memcpy(recvbuf, sendbuf, recvcount * dataSize[type]); } } void Serial::reduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code type, Operation::Code, size_t root) const { if (recvbuf != sendbuf && count > 0) { memcpy(recvbuf, sendbuf, count * dataSize[type]); } } void Serial::reduceInPlace(void*, size_t, Data::Code, Operation::Code, size_t root) const { return; } void Serial::allReduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code type, Operation::Code) const { if (recvbuf != sendbuf && count > 0) { memcpy(recvbuf, sendbuf, count * dataSize[type]); } } void Serial::allReduceInPlace(void*, size_t, Data::Code, Operation::Code) const { return; } void Serial::allGather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t, Data::Code type) const { if (recvbuf != sendbuf && sendcount > 0) { memcpy(recvbuf, sendbuf, sendcount * dataSize[type]); } } void Serial::allGatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int[], const int[], Data::Code type) const { if (recvbuf != sendbuf && sendcount > 0) { memcpy(recvbuf, sendbuf, sendcount * dataSize[type]); } } void Serial::allToAll(const void* sendbuf, size_t sendcount, void* recvbuf, size_t, Data::Code type) const { if (recvbuf != sendbuf && sendcount > 0) { memcpy(recvbuf, sendbuf, sendcount * dataSize[type]); } } void Serial::allToAllv(const void* sendbuf, const int sendcounts[], const int[], void* recvbuf, const int[], const int[], Data::Code type) const { if (recvbuf != sendbuf && sendcounts[0] > 0) { memcpy(recvbuf, sendbuf, sendcounts[0] * dataSize[type]); } } Status Serial::receive(void* recv, size_t count, Data::Code type, int /*source*/, int tag) const { AutoLock lock(SerialRequestPool::instance()); ReceiveRequest recv_request(recv, count, type, tag); SendRequest& send = SerialRequestPool::instance().matchingSendRequest(recv_request); if (tag != anyTag()) { ASSERT(tag == send.tag()); } ASSERT(count == send.count()); if (count > 0) { memcpy(recv, send.buffer(), send.count() * dataSize[send.type()]); } SerialStatus* st = new SerialStatus(); (*st).count_ = send.count(); (*st).source_ = 0; (*st).tag_ = send.tag(); (*st).error_ = 0; return Status(st); } Status Serial::sendReceiveReplace(void* sendrecv, size_t count, Data::Code type, int /*dest*/, int sendtag, int /*source*/, int recvtag) const { AutoLock lock(SerialRequestPool::instance()); SerialRequestPool::instance().createSendRequest(sendrecv, count, type, sendtag); ReceiveRequest recv_request(sendrecv, count, type, recvtag); SendRequest& send = SerialRequestPool::instance().matchingSendRequest(recv_request); if (recvtag != anyTag()) { ASSERT(recvtag == send.tag()); } ASSERT(count == send.count()); if (count > 0) { memcpy(sendrecv, send.buffer(), send.count() * dataSize[send.type()]); } SerialStatus* st = new SerialStatus(); (*st).count_ = send.count(); (*st).source_ = 0; (*st).tag_ = send.tag(); (*st).error_ = 0; return Status(st); } void Serial::send(const void* send, size_t count, Data::Code type, int /*dest*/, int tag) const { AutoLock lock(SerialRequestPool::instance()); SerialRequestPool::instance().createSendRequest(send, count, type, tag); } void Serial::synchronisedSend(const void* send, size_t count, Data::Code type, int /*dest*/, int tag) const { // TODO: see if this is good enough AutoLock lock(SerialRequestPool::instance()); SerialRequestPool::instance().createSendRequest(send, count, type, tag); } Request Serial::iReceive(void* recv, size_t count, Data::Code type, int /*source*/, int tag) const { AutoLock lock(SerialRequestPool::instance()); return SerialRequestPool::instance().createReceiveRequest(recv, count, type, tag); } Request Serial::iSend(const void* send, size_t count, Data::Code type, int /*dest*/, int tag) const { AutoLock lock(SerialRequestPool::instance()); return SerialRequestPool::instance().createSendRequest(send, count, type, tag); } Status Serial::createStatus() { return Status(new SerialStatus()); } Request Serial::request(int request) const { AutoLock lock(SerialRequestPool::instance()); return SerialRequestPool::instance()[request]; } Group Serial::group(int) const { NOTIMP; }; Group Serial::group() const { NOTIMP; }; Group Serial::remoteGroup() const { NOTIMP; }; Comm& Serial::create(const Group&, const std::string& name) const { NOTIMP; }; Comm& Serial::create(const Group&, int tag, const std::string& name) const { NOTIMP; }; void Serial::print(std::ostream& os) const { os << "Serial()"; } int Serial::communicator() const { return 0; } eckit::SharedBuffer Serial::broadcastFile(const PathName& filepath, size_t) const { std::unique_ptr dh(filepath.fileHandle()); Length len = dh->openForRead(); AutoClose closer(*dh); eckit::SharedBuffer buffer(len); dh->read(buffer->data(), len); if (not len) { throw ShortFile(filepath); } if (filepath.isDir()) { errno = EISDIR; throw CantOpenFile(filepath); } return buffer; } static CommBuilder SerialBuilder("serial"); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi eckit-2.0.7/src/eckit/mpi/Operation.h0000664000175000017500000000205315161702250017566 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_Operation_h #define eckit_mpi_Operation_h namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- struct Operation { enum Code { SUM = 0, PROD, MAX, MIN, MAXLOC, MINLOC, MAX_OPERATION_CODE, }; }; Operation::Code sum(); Operation::Code prod(); Operation::Code max(); Operation::Code min(); Operation::Code maxloc(); Operation::Code minloc(); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/Group.cc0000664000175000017500000001012515161702250017057 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/mpi/Group.h" #include "eckit/mpi/Comm.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class NullGroupContent : public GroupContent { public: virtual ~NullGroupContent() {} virtual void print(std::ostream& os) const { os << "NullGroup()"; } virtual int group() const { return -1; } virtual int compare(const GroupContent&) const { return -1; } virtual NullGroupContent* difference(const GroupContent&) const { return new NullGroupContent(); } virtual NullGroupContent* intersection(const GroupContent&) const { return new NullGroupContent(); } virtual NullGroupContent* union_group(const GroupContent&) const { return new NullGroupContent(); } virtual size_t size() const { return 0; } virtual int rank() const { return -1; } virtual NullGroupContent* excl(const std::vector& ranks) const { return new NullGroupContent(); }; virtual NullGroupContent* incl(const std::vector& ranks) const { return new NullGroupContent(); }; virtual NullGroupContent* range_excl(const std::vector>& ranks) const { return new NullGroupContent(); }; virtual NullGroupContent* range_incl(const std::vector>& ranks) const { return new NullGroupContent(); }; virtual std::unordered_map translate_ranks(const std::vector&, const GroupContent&) const { return std::unordered_map{}; }; }; //---------------------------------------------------------------------------------------------------------------------- Group::Group() : content_(new NullGroupContent()) { content_->attach(); } Group::Group(int group) : content_{nullptr} { *this = eckit::mpi::comm().group(group); } Group::Group(GroupContent* p) : content_(p) { content_->attach(); } Group::~Group() { content_->detach(); } Group::Group(const Group& s) : content_(s.content_) { content_->attach(); } Group& Group::operator=(const Group& s) { if (this == &s) { return *this; } if (content_) { content_->detach(); } content_ = s.content_; content_->attach(); return *this; } int Group::group() const { return content_->group(); } int Group::compare(const Group& other) const { return content_->compare(*other.content_); } Group Group::difference(const Group& other) const { return Group(content_->difference(*other.content_)); } Group Group::intersection(const Group& other) const { return Group(content_->intersection(*other.content_)); } Group Group::union_group(const Group& other) const { return Group(content_->union_group(*other.content_)); } size_t Group::size() const { return content_->size(); } int Group::rank() const { return content_->rank(); } Group Group::excl(const std::vector& ranks) const { return Group(content_->excl(ranks)); }; Group Group::incl(const std::vector& ranks) const { return Group(content_->incl(ranks)); }; Group Group::range_excl(const std::vector>& ranks) const { return Group(content_->range_excl(ranks)); }; Group Group::range_incl(const std::vector>& ranks) const { return Group(content_->range_incl(ranks)); }; std::unordered_map Group::translate_ranks(const std::vector& ranks, const Group& other) const { return content_->translate_ranks(ranks, *other.content_); }; void Group::print(std::ostream& out) const { content_->print(out); } GroupContent::~GroupContent() {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi eckit-2.0.7/src/eckit/mpi/Buffer.h0000664000175000017500000000253215161702250017041 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_Buffer_h #define eckit_mpi_Buffer_h #include #include namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- /// Buffer handles colleciton of vector pieces into a larger vector template > struct Buffer { using value_type = DATA_TYPE; using iterator = typename std::vector::iterator; int cnt; std::vector counts; std::vector displs; std::vector buffer; Buffer(size_t size) { counts.resize(size); displs.resize(size); } iterator begin() { return buffer.begin(); } iterator end() { return buffer.end(); } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/DataType.cc0000664000175000017500000001140015161702250017473 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/mpi/DataType.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- template <> const char* Data::Type::name() { return "byte"; } template <> Data::Code Data::Type::code() { return Data::BYTE; } template <> size_t Data::Type::size() { return 1; } template <> const char* Data::Type::name() { return "char"; } template <> Data::Code Data::Type::code() { return Data::CHAR; } template <> size_t Data::Type::size() { return sizeof(char); } template <> const char* Data::Type::name() { return "int"; } template <> Data::Code Data::Type::code() { return Data::INT; } template <> size_t Data::Type::size() { return sizeof(int); } template <> const char* Data::Type::name() { return "long"; } template <> Data::Code Data::Type::code() { return Data::LONG; } template <> size_t Data::Type::size() { return sizeof(long); } template <> const char* Data::Type::name() { return "long long"; } template <> Data::Code Data::Type::code() { return Data::LONG_LONG; } template <> size_t Data::Type::size() { return sizeof(long long); } template <> const char* Data::Type::name() { return "float"; } template <> Data::Code Data::Type::code() { return Data::FLOAT; } template <> size_t Data::Type::size() { return sizeof(float); } template <> const char* Data::Type::name() { return "double"; } template <> Data::Code Data::Type::code() { return Data::DOUBLE; } template <> size_t Data::Type::size() { return sizeof(double); } template <> const char* Data::Type::name() { return "size_t"; } template <> Data::Code Data::Type::code() { return Data::UNSIGNED_LONG; } template <> size_t Data::Type::size() { return sizeof(size_t); } //---------------------------------------------------------------------------------------------------------------------- template <> const char* Data::Type >::name() { return ""; } template <> Data::Code Data::Type >::code() { return Data::INT_INT; } template <> size_t Data::Type >::size() { return sizeof(std::pair); } template <> const char* Data::Type >::name() { return ""; } template <> Data::Code Data::Type >::code() { return Data::TWO_LONG; } template <> size_t Data::Type >::size() { return sizeof(std::pair); } template <> const char* Data::Type >::name() { return ""; } template <> Data::Code Data::Type >::code() { return Data::TWO_LONG_LONG; } template <> size_t Data::Type >::size() { return sizeof(std::pair); } template <> const char* Data::Type >::name() { return ""; } template <> Data::Code Data::Type >::code() { return Data::LONG_INT; } template <> size_t Data::Type >::size() { return sizeof(std::pair); } template <> const char* Data::Type >::name() { return ""; } template <> Data::Code Data::Type >::code() { return Data::FLOAT_INT; } template <> size_t Data::Type >::size() { return sizeof(std::pair); } template <> const char* Data::Type >::name() { return ""; } template <> Data::Code Data::Type >::code() { return Data::DOUBLE_INT; } template <> size_t Data::Type >::size() { return sizeof(std::pair); } template <> const char* Data::Type >::name() { return "complex"; } template <> Data::Code Data::Type >::code() { return Data::DOUBLE_COMPLEX; } template <> size_t Data::Type >::size() { return sizeof(std::complex); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi eckit-2.0.7/src/eckit/mpi/Serial.h0000664000175000017500000001316415161702250017052 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_Serial_h #define eckit_mpi_Serial_h #include "eckit/mpi/Comm.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class Serial : public eckit::mpi::Comm { public: struct Constants { static constexpr int anyTag() { return -1; } static constexpr int anySource() { return -1; } static constexpr int undefined() { return -32766; } static constexpr int procNull() { return -1; } }; protected: // methods template friend class CommBuilder; Serial(std::string_view name); Serial(std::string_view name, int); ~Serial() override; eckit::mpi::Comm* self() const override; std::string processorName() const override; size_t remoteSize() const override; void barrier() const override; Request iBarrier() const override; void abort(int errorcode = -1) const override; Status wait(Request&) const override; Status waitAny(std::vector&, int&) const override; std::vector waitAll(std::vector&) const override; Status probe(int source, int tag) const override; Status iProbe(int source, int tag) const override; int anySource() const override; int anyTag() const override; int undefined() const override; int procNull() const override; size_t getCount(Status& st, Data::Code type) const override; void broadcast(void* buffer, size_t count, Data::Code type, size_t root) const override; virtual void gather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type, size_t root) const override; virtual void scatter(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type, size_t root) const override; virtual void gatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int recvcounts[], const int displs[], Data::Code type, size_t root) const override; virtual void scatterv(const void* sendbuf, const int sendcounts[], const int displs[], void* recvbuf, size_t recvcount, Data::Code type, size_t root) const override; virtual void reduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code type, Operation::Code op, size_t root) const override; virtual void reduceInPlace(void* sendrecvbuf, size_t count, Data::Code type, Operation::Code op, size_t root) const override; virtual void allReduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code type, Operation::Code op) const override; void allReduceInPlace(void* sendrecvbuf, size_t count, Data::Code type, Operation::Code op) const override; virtual void allGather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type) const override; virtual void allGatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int recvcounts[], const int displs[], Data::Code type) const override; virtual void allToAll(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type) const override; virtual void allToAllv(const void* sendbuf, const int sendcounts[], const int sdispls[], void* recvbuf, const int recvcounts[], const int rdispls[], Data::Code type) const override; Status receive(void* recv, size_t count, Data::Code type, int source, int tag) const override; void send(const void* send, size_t count, Data::Code type, int dest, int tag) const override; void synchronisedSend(const void* send, size_t count, Data::Code type, int dest, int tag) const override; Request iReceive(void* recv, size_t count, Data::Code type, int source, int tag) const override; Request iSend(const void* send, size_t count, Data::Code type, int dest, int tag) const override; virtual Status sendReceiveReplace(void* sendrecv, size_t count, Data::Code type, int dest, int sendtag, int source, int recvtag) const override; Comm& split(int color, const std::string& name) const override; void free() override; eckit::SharedBuffer broadcastFile(const eckit::PathName& filepath, size_t root) const override; void print(std::ostream&) const override; Status status() const override { return createStatus(); } Request request(int tag) const override; // Not implemented - returns NullGroup Group group(int) const override; // Not implemented - returns NullGroup Group group() const override; // Not implemented - returns NullGroup Group remoteGroup() const override; // Not implemented Comm& create(const Group&, const std::string& name) const override; // Not implemented Comm& create(const Group&, int tag, const std::string& name) const override; static Status createStatus(); int communicator() const override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/ParallelStatus.cc0000664000175000017500000000165515161702250020733 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/mpi/ParallelStatus.h" namespace eckit { namespace mpi { //---------------------------------------------------------------------------------------------------------------------- void ParallelStatus::print(std::ostream& os) const { os << "ParallelStatus(" << "source=" << source() << ",tag=" << tag() << ",error=" << error() << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace mpi } // namespace eckit eckit-2.0.7/src/eckit/mpi/Comm.h0000664000175000017500000011531215161702250016524 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_Comm_h #define eckit_mpi_Comm_h #include #include #include #include #include #include #include "eckit/filesystem/PathName.h" #include "eckit/io/SharedBuffer.h" #include "eckit/mpi/Buffer.h" #include "eckit/mpi/DataType.h" #include "eckit/mpi/Group.h" #include "eckit/mpi/Operation.h" #include "eckit/mpi/Request.h" #include "eckit/mpi/Status.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class Environment; /// @returns the communicator registered with associated name, or default communicator when NULL is /// passed Comm& comm(std::string_view name = {}); Comm& self(); /// Set a communicator as default void setCommDefault(std::string_view name); /// Register a communicator comming from Fortran code void addComm(std::string_view name, int comm); /// Register an existing communicator void addComm(std::string_view name, Comm* comm); /// Unregister a communicator and call free() on the comm. /// @pre Comm is registered in the environment /// @note Cannot unregister 'world', 'self' or default communicators /// @throws SeriousBug if trying to unregister 'world', 'self' or default communicators, /// or any communicator aliasing MPI_COMM_WORLD or MPI_COMM_SELF with the parallel backend. void deleteComm(std::string_view name); /// Unregister a communicator without calling free() on the comm, e.g. if the registered comm is wrapping an MPI /// communicator managed by an external library /// @pre Comm is registered in the environment /// @note Cannot unregister 'world', 'self' or default communicators /// @throws SeriousBug if trying to unregister 'world', 'self' or default communicators void unregisterComm(std::string_view name); /// Check if a communicator is registered bool hasComm(std::string_view name); std::vector listComms(); /// Finalises all the comms that are registered /// /// @note This should not be necessary to be called, since all singletong Comms finalise themselves /// on destruction /// when application shutsdown. Currently there is a bug in OpenMPI on MacOSX (see ECKIT-166) /// that implies that MPI_Finalize must be called explicitly before exiting main(). This is /// the only current reason to use this function. void finaliseAllComms(); namespace detail { /// Assertions for eckit::mpi code /// Don't use directly in client code void Assert(int code, const char* msg, const char* file, int line, const char* func); template struct is_std_vector : std::false_type {}; template struct is_std_vector > : std::true_type {}; } // namespace detail //---------------------------------------------------------------------------------------------------------------------- class Comm { friend class Environment; public: // class methods static Comm& comm(std::string_view name = {}); Comm() = default; Comm(const Comm&) = delete; Comm& operator=(const Comm&) = delete; Comm(Comm&&) = delete; Comm& operator=(Comm&&) = delete; public: // methods std::string name() const { return name_; } /// @brief Returns name of processor according to MPI virtual std::string processorName() const = 0; /// @brief MPI rank of process in this communicator size_t rank() const { return rank_; } /// @brief MPI size of this communicator size_t size() const { return size_; } virtual size_t remoteSize() const = 0; /// @brief MPI barrier for this communicator virtual void barrier() const = 0; /// @brief MPI non-blocking barrier for this communicator virtual Request iBarrier() const = 0; /// @brief MPI abort for this communicator virtual void abort(int errorcode = -1) const = 0; /// @brief Wait for Request to be completed, ignoring the return status virtual Status wait(Request&) const = 0; /// @brief Wait for one request out of a vector of requests to finish /// /// @param[in] requests The requests that one will be waited for. /// @param[out] index The request index in requests that has been waited for. virtual Status waitAny(std::vector& requests, int& index) const = 0; /// @brief Wait for all given requests to finish virtual std::vector waitAll(std::vector&) const = 0; /// @brief Probe for incoming messages (blocking) virtual Status probe(int source, int tag) const = 0; /// @brief Probe for incoming messages (non-blocking) /// Use status.error() to check if there is an incoming message /// status.error() == 0 if there is an incoming message virtual Status iProbe(int source, int tag) const = 0; virtual int procNull() const = 0; virtual int undefined() const = 0; virtual int anySource() const = 0; virtual int anyTag() const = 0; virtual Status status() const = 0; virtual Request request(int) const = 0; virtual Group group(int) const = 0; virtual Group group() const = 0; virtual Group remoteGroup() const = 0; virtual Comm& create(const Group&, const std::string& name) const = 0; virtual Comm& create(const Group&, int tag, const std::string& name) const = 0; template size_t getCount(Status& status) const; /// /// Broadcast, pointer to data (also covers scalars) /// template void broadcast(T& value, size_t root) const; template void broadcast(T* first, T* last, size_t root) const; template void broadcast(T buffer[], size_t count, size_t root) const; template void broadcast(typename std::vector& v, size_t root) const; template void broadcast(Iter first, Iter last, size_t root) const; /// /// Gather methods to one root, equal data sizes per rank /// template void gather(CIter first, CIter last, Iter rfirst, Iter rlast, size_t root) const; template void gather(const T send, std::vector& recv, size_t root) const; template void gather(const std::vector& send, std::vector& recv, size_t root) const; /// /// Gather methods to one root, variable data sizes per rank /// template void gatherv(const Value* sendbuf, size_t sendcount, Value* recvbuf, const int recvcounts[], const int displs[], size_t root) const; template void gatherv(CIter first, CIter last, Iter rfirst, Iter rlast, const int recvcounts[], const int displs[], size_t root) const; template void gatherv(CIter first, CIter last, Iter rfirst, Iter rlast, const std::vector& recvcounts, const std::vector& displs, size_t root) const; template void gatherv(const std::vector& send, std::vector& recv, const std::vector& recvcounts, const std::vector& displs, size_t root) const; /// /// Scatter methods from one root /// template void scatter(const std::vector& send, T& recv, size_t root) const; template void scatter(const std::vector& send, std::vector& recv, size_t root) const; /// /// Scatter methods from one root, variable data sizes per rank, pointer to data (also covers /// scalar case) /// template void scatterv(const Value* sendbuf, const int sendcounts[], const int displs[], Value* recvbuf, int recvcount, size_t root) const; template void scatterv(CIter first, CIter last, const int sendcounts[], const int displs[], Iter rfirst, Iter rlast, size_t root) const; template void scatterv(CIter first, CIter last, const std::vector& sendcounts, const std::vector& displs, Iter rfirst, Iter rlast, size_t root) const; /// /// Reduce operations, separate buffers /// template void reduce(const T send, T& recv, Operation::Code op, size_t root) const; template void reduce(const T* send, T* recv, size_t count, Operation::Code op, size_t root) const; template void reduce(const std::vector& send, std::vector& recv, Operation::Code op, size_t root) const; /// /// Reduce operations, in place buffer /// template void reduceInPlace(T* sendrecvbuf, size_t count, Operation::Code op, size_t root) const; template void reduceInPlace(T& sendrecvbuf, Operation::Code op, size_t root) const; template void reduceInPlace(Iter first, Iter last, Operation::Code op, size_t root) const; /// /// All reduce operations, separate buffers /// template ::value>* = nullptr> void allReduce(const T send, T& recv, Operation::Code op) const; template void allReduce(const T* send, T* recv, size_t count, Operation::Code op) const; template void allReduce(const std::vector& send, std::vector& recv, Operation::Code op) const; /// /// All reduce operations, in place buffer /// template void allReduceInPlace(T* sendrecvbuf, size_t count, Operation::Code op) const; template void allReduceInPlace(T& sendrecvbuf, Operation::Code op) const; template void allReduceInPlace(Iter first, Iter last, Operation::Code op) const; /// /// Gather methods from all, equal data sizes per rank /// template void allGather(const T sendval, Iter rfirst, Iter rlast) const; /// /// Gather methods from all, variable data sizes per rank /// template void allGatherv(CIter first, CIter last, Iter recvbuf, const int recvcounts[], const int displs[]) const; template void allGatherv(CIter first, CIter last, mpi::Buffer& recv) const; /// /// All to all methods, fixed data size /// template ::value>* = nullptr> void allToAll(const std::vector& send, std::vector& recv) const; /// /// All to All, variable data size /// template void allToAllv(const T* sendbuf, const int sendcounts[], const int sdispls[], T* recvbuf, const int recvcounts[], const int rdispls[]) const; /// /// Non-blocking receive /// template Request iReceive(T* recv, size_t count, int source, int tag) const; template Request iReceive(T& recv, int source, int tag) const; /// /// Blocking receive /// template Status receive(T* recv, size_t count, int source, int tag) const; template Status receive(T& recv, int source, int tag) const; /// /// Blocking send /// template void send(const T* sendbuf, size_t count, int dest, int tag) const; template void send(const T& sendbuf, int dest, int tag) const; /// /// Blocking send, until message received /// template void synchronisedSend(const T* sendbuf, size_t count, int dest, int tag) const; template void synchronisedSend(const T& sendbuf, int dest, int tag) const; /// /// Non-blocking send /// template Request iSend(const T* sendbuf, size_t count, int dest, int tag) const; template Request iSend(const T& sendbuf, int dest, int tag) const; /// /// In place simultaneous send and receive /// template Status sendReceiveReplace(T* sendrecv, size_t count, int dest, int sendtag, int source, int recvtag) const; template Status sendReceiveReplace(T& sendrecv, int dest, int sendtag, int source, int recvtag) const; /// /// All to all of vector< vector<> > /// template void allToAll(const std::vector, A3>& sendvec, std::vector, A4>& recvvec) const; /// /// Read file on one rank, and broadcast /// virtual eckit::SharedBuffer broadcastFile(const eckit::PathName& filepath, size_t root) const = 0; /// @brief Split the communicator based on color & give the new communicator a name virtual Comm& split(int color, const std::string& name) const = 0; /// @brief The communicator virtual int communicator() const = 0; protected: // methods virtual size_t getCount(Status& status, Data::Code datatype) const = 0; virtual void broadcast(void* buffer, size_t count, Data::Code datatype, size_t root) const = 0; virtual void gather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code datatype, size_t root) const = 0; virtual void scatter(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code datatype, size_t root) const = 0; virtual void gatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int recvcounts[], const int displs[], Data::Code datatype, size_t root) const = 0; virtual void scatterv(const void* sendbuf, const int sendcounts[], const int displs[], void* recvbuf, size_t recvcount, Data::Code datatype, size_t root) const = 0; virtual void reduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code datatype, Operation::Code op, size_t root) const = 0; virtual void reduceInPlace(void* sendrecvbuf, size_t count, Data::Code datatype, Operation::Code op, size_t root) const = 0; virtual void allReduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code datatype, Operation::Code op) const = 0; virtual void allReduceInPlace(void* sendrecvbuf, size_t count, Data::Code datatype, Operation::Code op) const = 0; virtual void allGather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code datatype) const = 0; virtual void allGatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int recvcounts[], const int displs[], Data::Code datatype) const = 0; virtual void allToAll(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code datatype) const = 0; virtual void allToAllv(const void* sendbuf, const int sendcounts[], const int sdispls[], void* recvbuf, const int recvcounts[], const int rdispls[], Data::Code datatype) const = 0; virtual Status receive(void* recv, size_t count, Data::Code datatype, int source, int tag) const = 0; virtual void send(const void* send, size_t count, Data::Code datatype, int dest, int tag) const = 0; virtual void synchronisedSend(const void* send, size_t count, Data::Code datatype, int dest, int tag) const = 0; virtual Request iReceive(void* recv, size_t count, Data::Code datatype, int source, int tag) const = 0; virtual Request iSend(const void* send, size_t count, Data::Code datatype, int dest, int tag) const = 0; virtual Status sendReceiveReplace(void* sendrecv, size_t count, Data::Code datatype, int dest, int sendtag, int source, int recvtag) const = 0; /// @brief Call free on this communicator /// After calling this method, the communicator should not be used again /// This is protected so only Environment can call it, typically via the deleteComm() method virtual void free() = 0; /// @brief Creates a communicator to self virtual eckit::mpi::Comm* self() const = 0; private: // methods virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream& s, const Comm& o) { o.print(s); return s; } protected: // methods Comm(std::string_view name); virtual ~Comm(); std::string name_; size_t rank_; size_t size_; }; //---------------------------------------------------------------------------------------------------------------------- class CommFactory { friend class eckit::mpi::Environment; std::string builder_; virtual Comm* make(std::string_view name) = 0; virtual Comm* make(std::string_view name, int) = 0; protected: CommFactory(std::string_view builder); virtual ~CommFactory(); static Comm* build(std::string_view name, std::string_view builder); static Comm* build(std::string_view name, std::string_view builder, int); }; template class CommBuilder : public CommFactory { Comm* make(std::string_view name) override { return new T(name); } Comm* make(std::string_view name, int comm) override { return new T(name, comm); } public: CommBuilder(const std::string& builder) : CommFactory(builder) {} }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi //---------------------------------------------------------------------------------------------------------------------- #if defined(__clang_analyzer__) // By adding C assert, the analyzer can make assumptions on conditions, preventing false positive warnings #include #define ECKIT_MPI_ASSERT(a) \ eckit::mpi::detail::Assert(!(a), #a, __FILE__, __LINE__, __func__); \ assert(a) #else #define ECKIT_MPI_ASSERT(a) eckit::mpi::detail::Assert(!(a), #a, __FILE__, __LINE__, __func__) #endif template size_t eckit::mpi::Comm::getCount(Status& status) const { return getCount(status, Data::Type::code()); } /// /// Broadcast, pointer to data (also covers scalars) /// template void eckit::mpi::Comm::broadcast(T& value, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); T* p = &value; broadcast(p, p + 1, root); } template void eckit::mpi::Comm::broadcast(T* first, T* last, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); broadcast(first, (last - first), Data::Type::code(), root); } template void eckit::mpi::Comm::broadcast(T buffer[], size_t count, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); broadcast(buffer, count, Data::Type::code(), root); } template void eckit::mpi::Comm::broadcast(typename std::vector& v, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); broadcast(v.begin(), v.end(), root); } template void eckit::mpi::Comm::broadcast(Iter first, Iter last, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); typename std::iterator_traits::difference_type n = std::distance(first, last); Data::Code type = Data::Type::value_type>::code(); broadcast(&(*first), n, type, root); } /// /// Gather methods to one root, equal data sizes per rank /// template void eckit::mpi::Comm::gather(CIter first, CIter last, Iter rfirst, Iter rlast, size_t root) const { using diff_t = typename std::iterator_traits::difference_type; const size_t commsize = size(); ECKIT_MPI_ASSERT(commsize > 0); ECKIT_MPI_ASSERT(root < commsize); const diff_t sendcount = std::distance(first, last); const diff_t rsize = std::distance(rfirst, rlast); ECKIT_MPI_ASSERT(rsize % commsize == 0); /* receiving size is multiple of comm().size() */ const diff_t recvcount = rsize / commsize; ECKIT_MPI_ASSERT(sendcount == recvcount); using CValue = typename std::iterator_traits::value_type; using Value = typename std::iterator_traits::value_type; Data::Code ctype = Data::Type::code(); Data::Code type = Data::Type::code(); ECKIT_MPI_ASSERT(ctype == type); const CValue* sendbuf = (first != last) ? &(*first) : nullptr; Value* recvbuf = (rfirst != rlast) ? &(*rfirst) : nullptr; gather(sendbuf, sendcount, recvbuf, recvcount, type, root); } template void eckit::mpi::Comm::gather(const T send, std::vector& recv, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(commsize > 0); ECKIT_MPI_ASSERT(root < commsize); ECKIT_MPI_ASSERT(recv.size() % commsize == 0); /* receiving size is multiple of comm().size() */ size_t recvcount = recv.size() / commsize; size_t sendcount = 1; ECKIT_MPI_ASSERT(recvcount == sendcount); gather(&send, sendcount, recv.data(), recvcount, Data::Type::code(), root); } template void eckit::mpi::Comm::gather(const std::vector& send, std::vector& recv, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(commsize > 0); ECKIT_MPI_ASSERT(root < commsize); ECKIT_MPI_ASSERT(recv.size() % commsize == 0); /* receiving size is multiple of comm().size() */ size_t recvcount = recv.size() / commsize; ECKIT_MPI_ASSERT(send.size() == recvcount); gather(send.data(), send.size(), recv.data(), recvcount, Data::Type::code(), root); } /// /// Gather methods to one root, variable data sizes per rank /// template void eckit::mpi::Comm::gatherv(const Value* sendbuf, size_t sendcount, Value* recvbuf, const int recvcounts[], const int displs[], size_t root) const { gatherv(sendbuf, sendcount, recvbuf, recvcounts, displs, Data::Type::code(), root); } template void eckit::mpi::Comm::gatherv(CIter first, CIter last, Iter rfirst, Iter rlast, const int recvcounts[], const int displs[], size_t root) const { typename std::iterator_traits::difference_type sendcount = std::distance(first, last); // typename std::iterator_traits::difference_type recvsize = std::distance(rfirst, // rlast); using CValue = typename std::iterator_traits::value_type; using Value = typename std::iterator_traits::value_type; Data::Code ctype = Data::Type::code(); Data::Code type = Data::Type::code(); ECKIT_MPI_ASSERT(ctype == type); const CValue* sendbuf = (first != last) ? &(*first) : nullptr; Value* recvbuf = (rfirst != rlast) ? &(*rfirst) : nullptr; gatherv(sendbuf, sendcount, recvbuf, recvcounts, displs, type, root); } template void eckit::mpi::Comm::gatherv(CIter first, CIter last, Iter rfirst, Iter rlast, const std::vector& recvcounts, const std::vector& displs, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); ECKIT_MPI_ASSERT(recvcounts.size() == commsize); ECKIT_MPI_ASSERT(displs.size() == commsize); gatherv(first, last, rfirst, rlast, recvcounts.data(), displs.data(), root); } template void eckit::mpi::Comm::gatherv(const std::vector& send, std::vector& recv, const std::vector& recvcounts, const std::vector& displs, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); if (rank() == root) { ECKIT_MPI_ASSERT(recvcounts.size() == commsize); ECKIT_MPI_ASSERT(displs.size() == commsize); ECKIT_MPI_ASSERT(recv.size() >= displs[commsize - 1] + recvcounts[commsize - 1]); } gatherv(send.begin(), send.end(), recv.begin(), recv.end(), recvcounts.data(), displs.data(), root); } /// /// Scatter methods from one root /// template void eckit::mpi::Comm::scatter(const std::vector& send, T& recv, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(commsize > 0); ECKIT_MPI_ASSERT(root < commsize); ECKIT_MPI_ASSERT(send.size() % commsize == 0); size_t sendcount = send.size() / commsize; size_t recvcount = 1; ECKIT_MPI_ASSERT(recvcount == sendcount); scatter(send.data(), sendcount, &recv, recvcount, Data::Type::code(), root); } template void eckit::mpi::Comm::scatter(const std::vector& send, std::vector& recv, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(commsize > 0); ECKIT_MPI_ASSERT(root < commsize); ECKIT_MPI_ASSERT(send.size() % commsize == 0); size_t sendcount = send.size() / commsize; size_t recvcount = recv.size(); ECKIT_MPI_ASSERT(recvcount == sendcount); scatter(send.data(), sendcount, recv.data(), recvcount, Data::Type::code(), root); } /// /// Scatter methods from one root, variable data sizes per rank, pointer to data (also covers scalar /// case) /// template void eckit::mpi::Comm::scatterv(const Value* sendbuf, const int sendcounts[], const int displs[], Value* recvbuf, int recvcount, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); scatterv(sendbuf, sendcounts, displs, recvbuf, recvcount, Data::Type::code(), root); } template void eckit::mpi::Comm::scatterv(CIter first, CIter last, const int sendcounts[], const int displs[], Iter rfirst, Iter rlast, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); typename std::iterator_traits::difference_type recvcounts = std::distance(rfirst, rlast); // typename std::iterator_traits::difference_type sendsize = std::distance(first, last); using CValue = typename std::iterator_traits::value_type; using Value = typename std::iterator_traits::value_type; Data::Code ctype = Data::Type::code(); Data::Code type = Data::Type::code(); ECKIT_MPI_ASSERT(ctype == type); const CValue* sendbuf = (first != last) ? &(*first) : nullptr; Value* recvbuf = (rfirst != rlast) ? &(*rfirst) : nullptr; scatterv(sendbuf, sendcounts, displs, recvbuf, recvcounts, type, root); } template void eckit::mpi::Comm::scatterv(CIter first, CIter last, const std::vector& sendcounts, const std::vector& displs, Iter rfirst, Iter rlast, size_t root) const { size_t commsize = size(); ECKIT_MPI_ASSERT(root < commsize); ECKIT_MPI_ASSERT(sendcounts.size() == commsize); ECKIT_MPI_ASSERT(displs.size() == commsize); scatterv(first, last, sendcounts.data(), displs.data(), rfirst, rlast, root); } /// /// Reduce operations, separate buffers /// template void eckit::mpi::Comm::reduce(const T send, T& recv, Operation::Code op, size_t root) const { reduce(&send, &recv, 1, Data::Type::code(), op, root); } template void eckit::mpi::Comm::reduce(const T* send, T* recv, size_t count, Operation::Code op, size_t root) const { reduce(send, recv, count, Data::Type::code(), op, root); } template void eckit::mpi::Comm::reduce(const std::vector& send, std::vector& recv, Operation::Code op, size_t root) const { ECKIT_MPI_ASSERT(send.size() == recv.size()); reduce(send.data(), recv.data(), send.size(), Data::Type::code(), op, root); } /// /// Reduce operations, in place buffer /// template void eckit::mpi::Comm::reduceInPlace(T* sendrecvbuf, size_t count, Operation::Code op, size_t root) const { reduceInPlace(sendrecvbuf, count, Data::Type::code(), op, root); } template void eckit::mpi::Comm::reduceInPlace(T& sendrecvbuf, Operation::Code op, size_t root) const { reduceInPlace(&sendrecvbuf, 1, Data::Type::code(), op, root); } template void eckit::mpi::Comm::reduceInPlace(Iter first, Iter last, Operation::Code op, size_t root) const { typename std::iterator_traits::difference_type count = std::distance(first, last); Data::Code type = Data::Type::value_type>::code(); reduceInPlace(&(*first), count, type, op, root); } /// /// All reduce operations, separate buffers /// template ::value>*> void eckit::mpi::Comm::allReduce(const T send, T& recv, Operation::Code op) const { allReduce(&send, &recv, 1, Data::Type::code(), op); } template void eckit::mpi::Comm::allReduce(const T* send, T* recv, size_t count, Operation::Code op) const { allReduce(send, recv, count, Data::Type::code(), op); } template void eckit::mpi::Comm::allReduce(const std::vector& send, std::vector& recv, Operation::Code op) const { ECKIT_MPI_ASSERT(send.size() == recv.size()); allReduce(send.data(), recv.data(), send.size(), Data::Type::code(), op); } /// /// All reduce operations, in place buffer /// template void eckit::mpi::Comm::allReduceInPlace(T* sendrecvbuf, size_t count, Operation::Code op) const { allReduceInPlace(sendrecvbuf, count, Data::Type::code(), op); } template void eckit::mpi::Comm::allReduceInPlace(T& sendrecvbuf, Operation::Code op) const { allReduceInPlace(&sendrecvbuf, 1, Data::Type::code(), op); } template void eckit::mpi::Comm::allReduceInPlace(Iter first, Iter last, Operation::Code op) const { typename std::iterator_traits::difference_type count = std::distance(first, last); Data::Code type = Data::Type::value_type>::code(); allReduceInPlace(&(*first), count, type, op); } /// /// Gather methods from all, equal data sizes per rank /// template void eckit::mpi::Comm::allGather(T sendval, Iter rfirst, Iter rlast) const { Data::Code ctype = Data::Type::code(); Data::Code type = Data::Type::value_type>::code(); ECKIT_MPI_ASSERT(ctype == type); typename std::iterator_traits::difference_type rsize = std::distance(rfirst, rlast); ECKIT_MPI_ASSERT(rsize % size() == 0); size_t recvcount = rsize / size(); ECKIT_MPI_ASSERT(recvcount == 1); allGather(&sendval, 1, &(*rfirst), recvcount, type); } /// /// Gather methods from all, variable data sizes per rank /// template void eckit::mpi::Comm::allGatherv(CIter first, CIter last, Iter rfirst, const int recvcounts[], const int displs[]) const { typename std::iterator_traits::difference_type sendcount = std::distance(first, last); int recvcount = 0; int commsize = static_cast(size()); for (int i = 0; i < commsize; ++i) { recvcount += recvcounts[i]; } using Value = typename std::iterator_traits::value_type; Data::Code ctype = Data::Type::code(); Data::Code type = Data::Type::value_type>::code(); ECKIT_MPI_ASSERT(ctype == type); const Value* sendbuf = (sendcount > 0) ? &(*first) : nullptr; Value* recvbuf = (recvcount > 0) ? &(*rfirst) : nullptr; allGatherv(sendbuf, sendcount, recvbuf, recvcounts, displs, type); } /// /// All to all methods, fixed data size /// template ::value>*> void eckit::mpi::Comm::allToAll(const std::vector& send, std::vector& recv) const { size_t commsize = size(); ECKIT_MPI_ASSERT(commsize > 0); ECKIT_MPI_ASSERT(send.size() % commsize == 0); ECKIT_MPI_ASSERT(recv.size() % commsize == 0); size_t sendcount = send.size() / commsize; size_t recvcount = recv.size() / commsize; allToAll(send.data(), sendcount, recv.data(), recvcount, Data::Type::code()); } /// /// All to All, variable data size /// template void eckit::mpi::Comm::allToAllv(const T* sendbuf, const int sendcounts[], const int sdispls[], T* recvbuf, const int recvcounts[], const int rdispls[]) const { allToAllv(sendbuf, sendcounts, sdispls, recvbuf, recvcounts, rdispls, Data::Type::code()); } /// /// Non-blocking receive /// template eckit::mpi::Request eckit::mpi::Comm::iReceive(T* recv, size_t count, int source, int tag) const { return iReceive(recv, count, Data::Type::code(), source, tag); } template eckit::mpi::Request eckit::mpi::Comm::iReceive(T& recv, int source, int tag) const { return iReceive(&recv, 1, Data::Type::code(), source, tag); } /// /// Blocking receive /// template eckit::mpi::Status eckit::mpi::Comm::receive(T* recv, size_t count, int source, int tag) const { return receive(recv, count, Data::Type::code(), source, tag); } template eckit::mpi::Status eckit::mpi::Comm::receive(T& recv, int source, int tag) const { return receive(&recv, 1, Data::Type::code(), source, tag); } /// /// Blocking send /// template void eckit::mpi::Comm::send(const T* sendbuf, size_t count, int dest, int tag) const { send(sendbuf, count, Data::Type::code(), dest, tag); } template void eckit::mpi::Comm::send(const T& sendbuf, int dest, int tag) const { send(&sendbuf, 1, Data::Type::code(), dest, tag); } template void eckit::mpi::Comm::synchronisedSend(const T* sendbuf, size_t count, int dest, int tag) const { synchronisedSend(sendbuf, count, Data::Type::code(), dest, tag); } template void eckit::mpi::Comm::synchronisedSend(const T& sendbuf, int dest, int tag) const { synchronisedSend(&sendbuf, 1, Data::Type::code(), dest, tag); } /// /// In place simultaneous send and receive /// template eckit::mpi::Status eckit::mpi::Comm::sendReceiveReplace(T* sendrecv, size_t count, int dest, int sendtag, int source, int recvtag) const { return sendReceiveReplace(sendrecv, count, Data::Type::code(), dest, sendtag, source, recvtag); } template eckit::mpi::Status eckit::mpi::Comm::sendReceiveReplace(T& sendrecv, int dest, int sendtag, int source, int recvtag) const { return sendReceiveReplace(&sendrecv, 1, Data::Type::code(), dest, sendtag, source, recvtag); } /// /// Non-blocking send /// template eckit::mpi::Request eckit::mpi::Comm::iSend(const T* sendbuf, size_t count, int dest, int tag) const { return iSend(sendbuf, count, Data::Type::code(), dest, tag); } template eckit::mpi::Request eckit::mpi::Comm::iSend(const T& sendbuf, int dest, int tag) const { return iSend(&sendbuf, 1, Data::Type::code(), dest, tag); } template void eckit::mpi::Comm::allGatherv(CIter first, CIter last, mpi::Buffer& recv) const { int sendcnt = int(std::distance(first, last)); allGather(sendcnt, recv.counts.begin(), recv.counts.end()); recv.displs[0] = 0; recv.cnt = recv.counts[0]; size_t mpiSize = size(); for (size_t jpart = 1; jpart < mpiSize; ++jpart) { recv.displs[jpart] = recv.displs[jpart - 1] + recv.counts[jpart - 1]; recv.cnt += recv.counts[jpart]; } recv.buffer.resize(recv.cnt); allGatherv(first, last, recv.buffer.data(), recv.counts.data(), recv.displs.data()); } template void eckit::mpi::Comm::allToAll(const std::vector, A3>& sendvec, std::vector, A4>& recvvec) const { size_t commsize = size(); ECKIT_MPI_ASSERT(sendvec.size() == commsize); ECKIT_MPI_ASSERT(recvvec.size() == commsize); // Get send-information std::vector sendcounts(commsize); std::vector senddispls(commsize); int sendcnt; senddispls[0] = 0; sendcounts[0] = sendvec[0].size(); sendcnt = sendcounts[0]; for (size_t jproc = 1; jproc < commsize; ++jproc) { senddispls[jproc] = senddispls[jproc - 1] + sendcounts[jproc - 1]; sendcounts[jproc] = sendvec[jproc].size(); sendcnt += sendcounts[jproc]; } // Get recv-information std::vector recvcounts(commsize); std::vector recvdispls(commsize); int recvcnt; allToAll(sendcounts, recvcounts); recvdispls[0] = 0; recvcnt = recvcounts[0]; for (size_t jproc = 1; jproc < commsize; ++jproc) { recvdispls[jproc] = recvdispls[jproc - 1] + recvcounts[jproc - 1]; recvcnt += recvcounts[jproc]; } // Communicate std::vector sendbuf(sendcnt); std::vector recvbuf(recvcnt); int cnt = 0; for (size_t jproc = 0; jproc < commsize; ++jproc) { for (int i = 0; i < sendcounts[jproc]; ++i) { sendbuf[cnt++] = sendvec[jproc][i]; } } allToAllv(sendbuf.data(), sendcounts.data(), senddispls.data(), recvbuf.data(), recvcounts.data(), recvdispls.data()); cnt = 0; for (size_t jproc = 0; jproc < commsize; ++jproc) { recvvec[jproc].resize(recvcounts[jproc]); for (int i = 0; i < recvcounts[jproc]; ++i) { recvvec[jproc][i] = recvbuf[cnt++]; } } } //---------------------------------------------------------------------------------------------------------------------- #undef ECKIT_MPI_ASSERT #endif eckit-2.0.7/src/eckit/mpi/SerialStatus.h0000664000175000017500000000240115161702250020246 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_SerialStatus_h #define eckit_mpi_SerialStatus_h #include #include "eckit/mpi/Status.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class Serial; class SerialStatus : public StatusContent { SerialStatus(); private: // methods friend class Serial; int source() const override { return source_; } int tag() const override { return tag_; } int error() const override { return error_; } void print(std::ostream&) const override; private: // members int source_; int tag_; int error_; size_t count_; ///< counts number of elements transfered in comm }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/Request.h0000664000175000017500000000402015161702250017252 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_Request_h #define eckit_mpi_Request_h #include #include "eckit/memory/Counted.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- class RequestContent : public Counted { public: ~RequestContent() override; virtual void print(std::ostream&) const = 0; virtual int request() const = 0; virtual bool test() = 0; }; //---------------------------------------------------------------------------------------------------------------------- /// Request by construction has always a valid content_ /// @invariant content_ is not null class Request { public: // methods /// Null request constructor Request(); /// Request constructor from the Request() integer /// Use only for interfacing with Fortran Request(int); /// Constructor Request(RequestContent*); ~Request(); Request(const Request&); Request& operator=(const Request&); template T& as() { return dynamic_cast(*content_); } /// Returns this request interpreted as a int by the underlying implementation /// Use only for interfacing with Fortran int request() const; bool test(); private: // methods void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const Request& o) { o.print(s); return s; } private: // members RequestContent* content_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi #endif eckit-2.0.7/src/eckit/mpi/Status.cc0000664000175000017500000000264515161702250017256 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/mpi/Status.h" #include "eckit/exception/Exceptions.h" namespace eckit::mpi { //---------------------------------------------------------------------------------------------------------------------- Status::Status() : content_(new NullStatus()) { content_->attach(); } Status::Status(StatusContent* p) : content_(p) { ASSERT(p); content_->attach(); } Status::~Status() { content_->detach(); } Status::Status(const Status& s) : content_(s.content_) { content_->attach(); } Status& Status::operator=(const Status& s) { if (this == &s) { return *this; } content_->detach(); content_ = s.content_; content_->attach(); return *this; } StatusContent::~StatusContent() {} void NullStatus::print(std::ostream& os) const { os << "NullStatus(" << "source=" << source() << ",tag=" << tag() << ",error=" << error() << ")"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::mpi eckit-2.0.7/src/eckit/mpi/Parallel.h0000664000175000017500000001374515161702250017374 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_mpi_Parallel_h #define eckit_mpi_Parallel_h #define OMPI_SKIP_MPICXX 1 #define MPICH_SKIP_MPICXX 1 #include #include "eckit/mpi/Comm.h" namespace eckit { namespace mpi { //---------------------------------------------------------------------------------------------------------------------- class Parallel : public eckit::mpi::Comm { protected: // methods template friend class CommBuilder; Parallel(std::string_view name); Parallel(std::string_view name, MPI_Comm comm, bool); Parallel(std::string_view name, int comm); ~Parallel() override; eckit::mpi::Comm* self() const override; std::string processorName() const override; size_t remoteSize() const override; void barrier() const override; Request iBarrier() const override; void abort(int errorcode = -1) const override; Status wait(Request&) const override; Status waitAny(std::vector&, int&) const override; std::vector waitAll(std::vector&) const override; Status probe(int source, int tag) const override; Status iProbe(int source, int tag) const override; int anySource() const override; int anyTag() const override; int undefined() const override; int procNull() const override; size_t getCount(Status& st, Data::Code type) const override; void broadcast(void* buffer, size_t count, Data::Code type, size_t root) const override; virtual void gather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type, size_t root) const override; virtual void scatter(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type, size_t root) const override; virtual void gatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int recvcounts[], const int displs[], Data::Code type, size_t root) const override; virtual void scatterv(const void* sendbuf, const int sendcounts[], const int displs[], void* recvbuf, size_t recvcount, Data::Code type, size_t root) const override; virtual void reduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code type, Operation::Code op, size_t root) const override; virtual void reduceInPlace(void* sendrecvbuf, size_t count, Data::Code type, Operation::Code op, size_t root) const override; virtual void allReduce(const void* sendbuf, void* recvbuf, size_t count, Data::Code type, Operation::Code op) const override; void allReduceInPlace(void* sendrecvbuf, size_t count, Data::Code type, Operation::Code op) const override; virtual void allGather(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type) const override; virtual void allGatherv(const void* sendbuf, size_t sendcount, void* recvbuf, const int recvcounts[], const int displs[], Data::Code type) const override; virtual void allToAll(const void* sendbuf, size_t sendcount, void* recvbuf, size_t recvcount, Data::Code type) const override; virtual void allToAllv(const void* sendbuf, const int sendcounts[], const int sdispls[], void* recvbuf, const int recvcounts[], const int rdispls[], Data::Code type) const override; Status receive(void* recv, size_t count, Data::Code type, int source, int tag) const override; void send(const void* send, size_t count, Data::Code type, int dest, int tag) const override; void synchronisedSend(const void* send, size_t count, Data::Code type, int dest, int tag) const override; Request iReceive(void* recv, size_t count, Data::Code type, int source, int tag) const override; Request iSend(const void* send, size_t count, Data::Code type, int dest, int tag) const override; virtual Status sendReceiveReplace(void* sendrecv, size_t count, Data::Code type, int dest, int sendtag, int source, int recvtag) const override; eckit::SharedBuffer broadcastFile(const eckit::PathName& filepath, size_t root) const override; Comm& split(int color, const std::string& name) const override; void free() override; void print(std::ostream&) const override; Status status() const override { return createStatus(); } Request request(int) const override; Group group(int) const override; Group group() const override; Group remoteGroup() const override; Comm& create(const Group&, const std::string& name) const override; Comm& create(const Group&, int tag, const std::string& name) const override; int communicator() const override; public: // static methods static Status createStatus(); static MPI_Status* toStatus(Status&); static MPI_Request* toRequest(Request&); public: /// Access internal MPI_Comm. /// Warning! Do not use the return value to free or modify the MPI communicator! MPI_Comm MPIComm() const; private: // methods friend class ParallelGroup; // Groups should not call free if mpi has been finalized. Hence PrallelGroup needs to // query finalized() static void initialize(); static void finalize(); static bool initialized(); static bool finalized(); private: // members MPI_Comm comm_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace mpi } // namespace eckit #endif eckit-2.0.7/src/eckit/mpi/CMakeLists.txt0000664000175000017500000000230215161702250020212 0ustar alastairalastairlist( APPEND eckit_mpi_srcs Buffer.h Comm.cc Comm.h DataType.cc DataType.h Operation.cc Operation.h Request.cc Request.h Group.cc Group.h Serial.cc Serial.h SerialData.h SerialStatus.cc SerialStatus.h SerialRequest.cc SerialRequest.h Status.cc Status.h ) if( HAVE_MPI ) list( APPEND eckit_mpi_srcs Parallel.cc Parallel.h ParallelStatus.cc ParallelStatus.h ParallelRequest.cc ParallelRequest.h ParallelGroup.cc ParallelGroup.h ) set(eckit_mpi_defs ${MPI_C_DEFINITIONS} ) set(eckit_mpi_incs ${MPI_C_INCLUDE_PATH} ) set(eckit_mpi_libs ${MPI_C_LIBRARIES} ) endif() ecbuild_add_library( TARGET eckit_mpi TYPE SHARED INSTALL_HEADERS ALL HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/mpi SOURCES ${eckit_mpi_srcs} DEFINITIONS "${eckit_mpi_defs}" PUBLIC_INCLUDES "${eckit_mpi_incs}" PUBLIC_LIBS "${eckit_mpi_libs}" PUBLIC_LIBS eckit ) eckit-2.0.7/src/eckit/mpi/ParallelGroup.cc0000664000175000017500000001263215161702250020541 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/log/CodeLocation.h" #include "eckit/mpi/Parallel.h" #include "eckit/mpi/ParallelGroup.h" namespace eckit { namespace mpi { void MPICall(int code, std::string_view mpifunc, const eckit::CodeLocation& loc); #define MPI_CALL(a) MPICall(a, #a, Here()) //---------------------------------------------------------------------------------------------------------------------- ParallelGroup::~ParallelGroup() { if (valid_ && !Parallel::finalized()) { MPI_CALL(MPI_Group_free(&group_)); } } ParallelGroup::ParallelGroup() : valid_{false} {} ParallelGroup::ParallelGroup(MPI_Group group) : valid_{true}, group_(group) {} void ParallelGroup::print(std::ostream& os) const { os << "ParallelGroup()"; } int ParallelGroup::group() const { return MPI_Group_c2f(group_); } int ParallelGroup::compare(const GroupContent& otherContent) const { return compare(dynamic_cast(otherContent)); } int ParallelGroup::compare(const ParallelGroup& other) const { int groupCompare; MPI_CALL(MPI_Group_compare(group_, other.group_, &groupCompare)); switch (groupCompare) { case MPI_IDENT: return 0; case MPI_SIMILAR: return 1; default: return -1; } } GroupContent* ParallelGroup::difference(const GroupContent& otherContent) const { return difference(dynamic_cast(otherContent)); }; ParallelGroup* ParallelGroup::difference(const ParallelGroup& other) const { MPI_Group newGroup; MPI_CALL(MPI_Group_difference(group_, other.group_, &newGroup)); return new ParallelGroup(newGroup); }; GroupContent* ParallelGroup::intersection(const GroupContent& otherContent) const { return intersection(dynamic_cast(otherContent)); }; ParallelGroup* ParallelGroup::intersection(const ParallelGroup& other) const { MPI_Group newGroup; MPI_CALL(MPI_Group_intersection(group_, other.group_, &newGroup)); return new ParallelGroup(newGroup); }; GroupContent* ParallelGroup::union_group(const GroupContent& otherContent) const { return union_group(dynamic_cast(otherContent)); }; ParallelGroup* ParallelGroup::union_group(const ParallelGroup& other) const { MPI_Group newGroup; MPI_CALL(MPI_Group_union(group_, other.group_, &newGroup)); return new ParallelGroup(newGroup); }; size_t ParallelGroup::size() const { int s; MPI_CALL(MPI_Group_size(group_, &s)); return ((size_t)s); }; int ParallelGroup::rank() const { int r; MPI_CALL(MPI_Group_rank(group_, &r)); return r; }; GroupContent* ParallelGroup::excl(const std::vector& ranks) const { MPI_Group newGroup; MPI_CALL(MPI_Group_excl(group_, ranks.size(), ranks.data(), &newGroup)); return new ParallelGroup(newGroup); } GroupContent* ParallelGroup::incl(const std::vector& ranks) const { MPI_Group newGroup; MPI_CALL(MPI_Group_incl(group_, ranks.size(), ranks.data(), &newGroup)); return new ParallelGroup(newGroup); } GroupContent* ParallelGroup::range_excl(const std::vector>& ranks) const { MPI_Group newGroup; int(*a)[3] = new int[ranks.size()][3]; for (int i = 0; i < ranks.size(); ++i) { a[i][0] = ranks[i][0]; a[i][1] = ranks[i][1]; a[i][2] = ranks[i][2]; } MPI_CALL(MPI_Group_range_excl(group_, ranks.size(), a, &newGroup)); return new ParallelGroup(newGroup); } GroupContent* ParallelGroup::range_incl(const std::vector>& ranks) const { MPI_Group newGroup; int(*a)[3] = new int[ranks.size()][3]; for (int i = 0; i < ranks.size(); ++i) { a[i][0] = ranks[i][0]; a[i][1] = ranks[i][1]; a[i][2] = ranks[i][2]; } MPI_CALL(MPI_Group_range_incl(group_, ranks.size(), a, &newGroup)); return new ParallelGroup(newGroup); } std::vector ParallelGroup::translate_ranks_native(const std::vector& ranks, const ParallelGroup& other) const { std::vector newVector(ranks.size()); MPI_CALL(MPI_Group_translate_ranks(group_, ranks.size(), ranks.data(), other.group_, newVector.data())); return newVector; } std::vector ParallelGroup::translate_ranks_native(const std::vector& ranks, const GroupContent& other) const { return translate_ranks_native(ranks, dynamic_cast(other)); } std::unordered_map ParallelGroup::translate_ranks(const std::vector& ranks, const GroupContent& other) const { std::unordered_map map; std::vector translated = translate_ranks_native(ranks, other); for (int r = 0; r < ranks.size(); ++r) { if (translated[r] != MPI_UNDEFINED) { map.emplace(ranks[r], translated[r]); } } return map; } //---------------------------------------------------------------------------------------------------------------------- } // namespace mpi } // namespace eckit eckit-2.0.7/src/eckit/persist/0000775000175000017500000000000015161702250016361 5ustar alastairalastaireckit-2.0.7/src/eckit/persist/Isa.h0000664000175000017500000000456415161702250017257 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_persist_Isa_h #define eckit_persist_Isa_h #include class TypeInfo; namespace eckit { class Isa { public: Isa* next_; TypeInfo* type_; Isa(TypeInfo* t, Isa* n) : next_(n), type_(t) {} static void add(TypeInfo* t, const std::string&); static Isa* get(const std::string&); }; class Schema { public: virtual ~Schema() {} virtual void start(const std::string&, size_t size) = 0; virtual void member(const std::string&, size_t size, size_t offset, const std::string& type) = 0; virtual void end(const std::string&) = 0; }; template void _describe(std::ostream& s, int depth, const T& what) { what.describe(s, depth); } void _describe(std::ostream& s, int depth, int what); void _describe(std::ostream& s, int depth, unsigned int what); void _describe(std::ostream& s, int depth, short what); void _describe(std::ostream& s, int depth, bool what); void _describe(std::ostream& s, int depth, unsigned short what); void _describe(std::ostream& s, int depth, long what); void _describe(std::ostream& s, int depth, long long what); void _describe(std::ostream& s, int depth, unsigned long long what); void _describe(std::ostream& s, int depth, unsigned long what); void _describe(std::ostream& s, int depth, char what); void _describe(std::ostream& s, int depth, unsigned char what); void _describe(std::ostream& s, int depth, double what); void _startClass(std::ostream& s, int depth, const std::string& name); void _endClass(std::ostream& s, int depth, const std::string& name); void _startMember(std::ostream& s, int depth, const std::string& name); void _endMember(std::ostream& s, int depth, const std::string& name); template void _describe(std::ostream& s, int depth, const std::string& name, const T& what) { _startMember(s, depth, name); _describe(s, depth, what); _endMember(s, depth, name); } } // namespace eckit #endif eckit-2.0.7/src/eckit/persist/DumpLoad.h0000664000175000017500000001126715161702250020246 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File DumpLoad.h // Baudouin Raoult - ECMWF Sep 99 #ifndef eckit_DumpLoad_h #define eckit_DumpLoad_h #include namespace eckit { //-------------------------------------------------------------------------------------------------- class DumpLoad { public: // -- Exceptions // None // -- Contructors DumpLoad(); // -- Destructor virtual ~DumpLoad(); // -- Convertors // None // -- Operators // None // -- Methods virtual void beginObject(const std::string&) = 0; virtual void endObject() = 0; virtual void nullObject() = 0; virtual std::string nextObject() = 0; virtual void doneObject() = 0; virtual void reset() = 0; // -------------------- virtual void load(std::string&) = 0; virtual void load(float&) = 0; virtual void load(double&) = 0; virtual void load(int&) = 0; virtual void load(unsigned int&) = 0; virtual void load(long&) = 0; virtual void load(unsigned long&) = 0; virtual void load(long long&) = 0; virtual void load(unsigned long long&) = 0; virtual void load(char&) = 0; virtual void load(unsigned char&) = 0; // -- Dump virtual void dump(const std::string&) = 0; virtual void dump(float) = 0; virtual void dump(double) = 0; virtual void dump(int) = 0; virtual void dump(unsigned int) = 0; virtual void dump(long) = 0; virtual void dump(unsigned long) = 0; virtual void dump(long long) = 0; virtual void dump(unsigned long long) = 0; virtual void dump(char) = 0; virtual void dump(unsigned char) = 0; // -- virtual void push(const std::string&, const std::string&) = 0; virtual std::string get(const std::string&) = 0; virtual void pop(const std::string&) = 0; // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods // void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed DumpLoad(const DumpLoad&); DumpLoad& operator=(const DumpLoad&); // -- Members // None // -- Methods // None // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends // friend std::ostream& operator<<(std::ostream& s,const DumpLoad& p) // { p.print(s); return s; } }; //-- Load template inline void load(DumpLoad& a, T& b) { b.load(a); } inline void load(DumpLoad& a, std::string& b) { a.load(b); } inline void load(DumpLoad& a, float& b) { a.load(b); } inline void load(DumpLoad& a, double& b) { a.load(b); } inline void load(DumpLoad& a, int& b) { a.load(b); } inline void load(DumpLoad& a, unsigned int& b) { a.load(b); } inline void load(DumpLoad& a, long& b) { a.load(b); } inline void load(DumpLoad& a, unsigned long& b) { a.load(b); } inline void load(DumpLoad& a, long long& b) { a.load(b); } inline void load(DumpLoad& a, unsigned long long& b) { a.load(b); } inline void load(DumpLoad& a, char& b) { a.load(b); } inline void load(DumpLoad& a, unsigned char& b) { a.load(b); } // -- Dump template inline void dump(DumpLoad& a, const T& b) { b.dump(a); } inline void dump(DumpLoad& a, const std::string& b) { a.dump(b); } inline void dump(DumpLoad& a, float b) { a.dump(b); } inline void dump(DumpLoad& a, double b) { a.dump(b); } inline void dump(DumpLoad& a, int b) { a.dump(b); } inline void dump(DumpLoad& a, unsigned int b) { a.dump(b); } inline void dump(DumpLoad& a, long b) { a.dump(b); } inline void dump(DumpLoad& a, unsigned long b) { a.dump(b); } inline void dump(DumpLoad& a, long long b) { a.dump(b); } inline void dump(DumpLoad& a, unsigned long long b) { a.dump(b); } inline void dump(DumpLoad& a, char b) { a.dump(b); } inline void dump(DumpLoad& a, unsigned char b) { a.dump(b); } //-------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/persist/Bless.h0000664000175000017500000001161415161702250017605 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_persist_Bless_h #define eckit_persist_Bless_h #include "eckit/persist/Isa.h" //-------------------------------------------------------------------------------------------------- #ifdef __GNUC__ /* GCC gets confused about offsetof */ static char _offset_dummy[80]; static void* _offset = &_offset_dummy; #define member_offset(Z, z) \ size_t(reinterpret_cast(&reinterpret_cast(_offset)->z) - reinterpret_cast(_offset)) #define member_size(Z, z) size_t(sizeof(reinterpret_cast(_offset)->z)) namespace eckit { namespace { static void* keep_gcc_quiet_about_offset_2(void* d); static void* keep_gcc_quiet_about_offset_1(void*) { return keep_gcc_quiet_about_offset_2(_offset); } static void* keep_gcc_quiet_about_offset_2(void*) { return keep_gcc_quiet_about_offset_1(_offset); } } // namespace } // namespace eckit #endif /* __GNUC__ */ #ifndef member_size #define member_size(a, b) size_t(sizeof(((a*)0)->b)) #endif #ifndef member_offset #define member_offset(a, b) size_t(&(((a*)0)->b)) #endif //-------------------------------------------------------------------------------------------------- namespace eckit { class Exporter; //-------------------------------------------------------------------------------------------------- class Bless { public: bool operator()(bool* a) { return *a; } int operator()(int* a) { return *a; } short operator()(short* a) { return *a; } char operator()(char* a) { return *a; } long operator()(long* a) { return *a; } long long operator()(long long* a) { return *a; } unsigned long operator()(unsigned long* a) { return *a; } unsigned int operator()(unsigned int* a) { return *a; } unsigned char operator()(unsigned char* a) { return *a; } unsigned short operator()(unsigned short* a) { return *a; } unsigned long long operator()(unsigned long long* a) { return *a; } double operator()(double* a) { return *a; } template Bless& operator()(T*) { return *this; } }; //-------------------------------------------------------------------------------------------------- class Evolve { public: Evolve(eckit::Exporter&); Evolve(Evolve*, const char*, const char*); Evolve operator()(const char*, const char* = nullptr); operator bool(); operator double(); operator int(); operator short(); operator char(); operator long(); operator long long(); operator unsigned int(); operator unsigned short(); operator unsigned char(); operator unsigned long(); operator unsigned long long(); const std::string& path() const { return path_; } private: Exporter& e_; std::string path_; Evolve* parent_; }; void _startObject(eckit::Exporter&, unsigned long long type, unsigned long long location, unsigned long long id, size_t count); void _endObject(eckit::Exporter&, unsigned long long type, unsigned long long location, unsigned long long id, size_t count); void _startSubObject(eckit::Exporter&); void _endSubObject(eckit::Exporter&); void _nextSubObject(eckit::Exporter&); void _startClass(eckit::Exporter&, const std::string& name); void _endClass(eckit::Exporter&, const std::string& name); void _startClass(eckit::Exporter&, const char* name); void _endClass(eckit::Exporter&, const char* name); void _startMember(eckit::Exporter&, const char* name); void _endMember(eckit::Exporter&, const char* name); //-------------------------------------------------------------------------------------------------- template void _export(eckit::Exporter& h, const T& what) { what._export(h); } void _export(eckit::Exporter&, int what); void _export(eckit::Exporter&, unsigned int what); void _export(eckit::Exporter&, short what); void _export(eckit::Exporter&, bool what); void _export(eckit::Exporter&, unsigned short what); void _export(eckit::Exporter&, long what); void _export(eckit::Exporter&, long long what); void _export(eckit::Exporter&, unsigned long long what); void _export(eckit::Exporter&, unsigned long what); void _export(eckit::Exporter&, char what); void _export(eckit::Exporter&, unsigned char what); void _export(eckit::Exporter&, double what); template void _export(eckit::Exporter& s, const char* name, const T& what) { _startMember(s, name); _export(s, what); _endMember(s, name); } //-------------------------------------------------------------------------------------------------- } // namespace eckit void* operator new(size_t, void* addr, eckit::Evolve&); #endif eckit-2.0.7/src/eckit/persist/DumpLoad.cc0000664000175000017500000000157015161702250020400 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/persist/DumpLoad.h" //---------------------------------------------------------------------------------------------------------------------- namespace eckit { //---------------------------------------------------------------------------------------------------------------------- DumpLoad::DumpLoad() {} DumpLoad::~DumpLoad() {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/persist/Isa.cc0000664000175000017500000000551415161702250017411 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "Isa.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static std::map* map_ = nullptr; void Isa::add(TypeInfo* t, const std::string& s) { Isa* i = new Isa(t, get(s)); // std::cout << "add isa " << s << " for " << t << std::endl; (*map_)[s] = i; } Isa* Isa::get(const std::string& s) { if (map_ == nullptr) { map_ = new std::map; } std::map::iterator j = map_->find(s); return (j == map_->end()) ? nullptr : (Isa*)(*j).second; } void _describe(std::ostream& s, int depth, int what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, unsigned int what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, short what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, bool what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, unsigned short what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, long what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, long long what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, unsigned long long what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, unsigned long what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, char what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, unsigned char what) { s << what << std::endl; } void _describe(std::ostream& s, int depth, double what) { s << what << std::endl; } void _startClass(std::ostream& s, int depth, const std::string& name) { for (int i = 0; i < depth; i++) { s << " "; } s << name; s << "{" << std::endl; } void _endClass(std::ostream& s, int depth, const std::string& name) { for (int i = 0; i < depth; i++) { s << " "; } s << "}" << std::endl; } void _startMember(std::ostream& s, int depth, const std::string& name) { for (int i = 0; i < depth; i++) { s << " "; } s << name << ": "; } void _endMember(std::ostream& s, int depth, const std::string& name) { // s << std::endl; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/persist/Exporter.h0000664000175000017500000001072515161702250020347 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Exporter.h // Baudouin Raoult - (c) ECMWF Oct 10 #ifndef eckit_Exporter_h #define eckit_Exporter_h #include #include namespace eckit { //----------------------------------------------------------------------------- class DataHandle; class Exporter { class Datatype { char type_; bool used_; double double_; long long signed_; unsigned long long unsigned_; public: Datatype(); Datatype(double d); Datatype(long long d); Datatype(unsigned long long d); operator double(); operator long long(); operator unsigned long long(); bool empty() const { return type_ == 0; } bool used() const { return used_; } void print(std::ostream& s) const; friend std::ostream& operator<<(std::ostream& s, const Datatype& p) { p.print(s); return s; } }; public: // -- Exceptions // None // -- Contructors Exporter(DataHandle&); Exporter(const Exporter&) = delete; Exporter& operator=(const Exporter&) = delete; Exporter(Exporter&&) = delete; Exporter& operator=(Exporter&&) = delete; // -- Destructor ~Exporter(); // -- Convertors // None // -- Operators // None // -- Methods void openForWrite(); void close(); DataHandle& handle() const { return handle_; } void startSchemas(); void endSchemas(); void startDatabases(); void endDatabases(); void startDatabase(const std::string&, unsigned long, unsigned long long); void endDatabase(const std::string&, unsigned long); void writeTag(char); char readTag(); void writeSigned(long long); void writeUnsigned(unsigned long long); void writeString(const std::string&); void writeString(const char*); void writeDouble(double); void dataBase(unsigned long long, const std::string&); void typeInfo(unsigned long long, const std::string&); long long readSigned(); unsigned long long readUnsigned(); std::string readString(); double readDouble(); bool nextDatabase(std::string& name, unsigned long long& id, unsigned long long& count); size_t nextObject(); void endObject(); void nextSubObject(); unsigned long long type() const { return type_; } void type(unsigned long long t) { type_ = t; } unsigned long long location() const { return location_; } unsigned long long objectId() const { return objectId_; } void startObject(unsigned long long, unsigned long long, unsigned long long, size_t); void endObject(unsigned long long, unsigned long long, unsigned long long, size_t); void startSubObject(); void endSubObject(); unsigned long long getUnsignedMember(const std::string&); long long getSignedMember(const std::string&); double getDoubleMember(const std::string&); // -- Overridden methods // None // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // members DataHandle& handle_; unsigned long long type_; unsigned long long location_; unsigned long long objectId_; unsigned long long objectCount_; unsigned long long subCount_; std::vector stack_; std::map members_; bool inObject_; // -- Methods long long _readSigned(); unsigned long long _readUnsigned(); std::string _readString(); double _readDouble(); std::string path() const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None // -- Friends friend std::ostream& operator<<(std::ostream& s, const Exporter& p) { p.print(s); return s; } }; //----------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/persist/Exporter.cc0000664000175000017500000004136315161702250020507 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/io/DataHandle.h" #include "eckit/persist/Exporter.h" //---------------------------------------------------------------------------------------------------------------------- void* operator new(size_t, void* addr, eckit::Evolve&) { return addr; } namespace eckit { //---------------------------------------------------------------------------------------------------------------------- void _export(eckit::Exporter& out, unsigned char what) { out.writeUnsigned(what); } void _export(eckit::Exporter& out, char what) { out.writeSigned(what); } void _export(eckit::Exporter& out, unsigned int what) { out.writeUnsigned(what); } void _export(eckit::Exporter& out, int what) { out.writeSigned(what); } void _export(eckit::Exporter& out, unsigned long what) { out.writeUnsigned(what); } void _export(eckit::Exporter& out, long what) { out.writeSigned(what); } void _export(eckit::Exporter& out, unsigned short what) { out.writeUnsigned(what); } void _export(eckit::Exporter& out, short what) { out.writeSigned(what); } void _export(eckit::Exporter& out, unsigned long long what) { out.writeUnsigned(what); } void _export(eckit::Exporter& out, long long what) { out.writeSigned(what); } void _export(eckit::Exporter& out, double what) { out.writeDouble(what); } //---------------------------------------------------------------------------------------------------------------------- const int MAX_STRING_LEN = 10240; enum { TAG_EOF = 'X', TAG_START_CLASS = 'C', TAG_END_CLASS = 'c', TAG_START_OBJECT = 'O', TAG_END_OBJECT = 'o', TAG_START_MEMBER = 'M', TAG_END_MEMBER = 'm', TAG_START_SUBOBJECT = 'L', TAG_END_SUBOBJECT = 'l', TAG_UNSIGNED = 'u', TAG_SIGNED = 's', TAG_STRING = 'S', TAG_DOUBLE = 'D', TAG_START_SCHEMAS = '{', TAG_END_SCHEMAS = '}', TAG_START_DATABASE = '[', TAG_END_DATABASE = ']' }; template struct Swap { static const int half = sizeof(T) >> 1; static const int last = sizeof(T) - 1; T operator()(T v) { unsigned char* p = (unsigned char*)&v; for (int i = 0; i < half; i++) { std::swap(p[i], p[last - i]); } return v; } }; // xlc needs this, // others will complain about double initialization #if defined(__xlC__) template const int Swap::half = sizeof(T) >> 1; template const int Swap::last = sizeof(T) - 1; #endif class Endian { public: #if eckit_LITTLE_ENDIAN template static T transform(T x) { return Swap()(x); } #else template static T transform(T x) { return x; } #endif }; Exporter::Exporter(DataHandle& handle) : handle_(handle), type_(0), location_(0), objectId_(0), objectCount_(0), subCount_(0), inObject_(false) {} Exporter::~Exporter() {} void Exporter::writeTag(char tag) { ASSERT(sizeof(tag) == 1); ASSERT(handle_.write(&tag, 1) == 1); } char Exporter::readTag() { char tag; ASSERT(sizeof(tag) == 1); ASSERT(handle_.read(&tag, 1) == 1); return tag; } void Exporter::writeString(const std::string& s) { size_t len = s.length(); char buffer[MAX_STRING_LEN]; if (len) { ASSERT(sizeof(s[0]) == 1); ASSERT(len <= sizeof(buffer)); } writeTag(TAG_STRING); writeUnsigned(len); for (size_t i = 0; i < len; i++) { buffer[i] = s[i]; } ASSERT((size_t)handle_.write(buffer, len) == len); } void Exporter::writeString(const char* s) { size_t len = strlen(s); writeTag(TAG_STRING); writeUnsigned(len); ASSERT(sizeof(s[0]) == 1); ASSERT((size_t)handle_.write(s, len) == len); } std::string Exporter::_readString() { try { std::string s; size_t len = readUnsigned(); for (size_t i = 0; i < len; i++) { char c; ASSERT(handle_.read(&c, 1) == 1); s += c; } return s; } catch (...) { std::cout << "Error reading std::string " << *this << std::endl; throw; } } std::string Exporter::readString() { ASSERT(readTag() == TAG_STRING); return _readString(); } void Exporter::writeDouble(double d) { writeTag(TAG_DOUBLE); ASSERT(sizeof(double) == 8); d = Endian::transform(d); ASSERT(handle_.write(&d, sizeof(d)) == sizeof(d)); } double Exporter::_readDouble() { double d; ASSERT(sizeof(double) == 8); ASSERT(handle_.read(&d, sizeof(d)) == sizeof(d)); return Endian::transform(d); } double Exporter::readDouble() { ASSERT(readTag() == TAG_DOUBLE); return _readDouble(); } void Exporter::writeSigned(long long d) { writeTag(TAG_SIGNED); ASSERT(sizeof(long long) == 8); d = Endian::transform(d); ASSERT(handle_.write(&d, sizeof(d)) == sizeof(d)); } long long Exporter::_readSigned() { long long d; ASSERT(sizeof(long long) == 8); ASSERT(handle_.read(&d, sizeof(d)) == sizeof(d)); return Endian::transform(d); } long long Exporter::readSigned() { ASSERT(readTag() == TAG_SIGNED); return _readSigned(); } void Exporter::writeUnsigned(unsigned long long d) { writeTag(TAG_UNSIGNED); ASSERT(sizeof(unsigned long long) == 8); d = Endian::transform(d); ASSERT(handle_.write(&d, sizeof(d)) == sizeof(d)); } unsigned long long Exporter::_readUnsigned() { unsigned long long d; ASSERT(sizeof(unsigned long long) == 8); ASSERT(handle_.read(&d, sizeof(d)) == sizeof(d)); return Endian::transform(d); } unsigned long long Exporter::readUnsigned() { ASSERT(readTag() == TAG_UNSIGNED); return _readUnsigned(); } void _startClass(eckit::Exporter& out, const std::string& name) { out.writeTag(TAG_START_CLASS); out.writeString(name); } void _endClass(eckit::Exporter& out, const std::string& name) { out.writeTag(TAG_END_CLASS); // out.writeString(name); } void _startClass(eckit::Exporter& out, const char* name) { out.writeTag(TAG_START_CLASS); out.writeString(name); } void _endClass(eckit::Exporter& out, const char* name) { out.writeTag(TAG_END_CLASS); // out.writeString(name); } void _startMember(eckit::Exporter& out, const char* name) { out.writeTag(TAG_START_MEMBER); out.writeString(name); } void _endMember(eckit::Exporter& out, const char* name) { out.writeTag(TAG_END_MEMBER); // out.writeString(name); } void Exporter::startObject(unsigned long long type, unsigned long long location, unsigned long long id, size_t count) { writeTag(TAG_START_OBJECT); writeUnsigned(type); writeUnsigned(location); writeUnsigned(id); writeUnsigned(count); objectCount_++; ASSERT(!inObject_); inObject_ = true; subCount_ = 0; } void _startObject(eckit::Exporter& e, unsigned long long type, unsigned long long location, unsigned long long id, size_t count) { e.startObject(type, location, id, count); } void Exporter::endObject(unsigned long long type, unsigned long long location, unsigned long long id, size_t count) { writeTag(TAG_END_OBJECT); ASSERT(inObject_); inObject_ = false; ASSERT(subCount_); /* writeUnsigned(type); writeUnsigned(location); writeUnsigned(id); writeUnsigned(count); */ } void Exporter::startSubObject() { writeTag(TAG_START_SUBOBJECT); subCount_++; } void _startSubObject(eckit::Exporter& e) { e.startSubObject(); } void Exporter::endSubObject() { writeTag(TAG_END_SUBOBJECT); } void _endSubObject(eckit::Exporter& e) { e.endSubObject(); } void _endObject(eckit::Exporter& e, unsigned long long type, unsigned long long location, unsigned long long id, size_t count) { e.endObject(type, location, id, count); } void Exporter::endObject() { ASSERT(readTag() == TAG_END_OBJECT); ASSERT(subCount_); for (std::map::iterator j = members_.begin(); j != members_.end(); ++j) { if (!(*j).second.used()) { std::cout << "WARNING NOT USED [" << (*j).first << "]" << std::endl; } } members_.clear(); stack_.clear(); } bool Exporter::nextDatabase(std::string& name, unsigned long long& id, unsigned long long& count) { char tag = readTag(); if (tag == TAG_EOF) { return false; } if (tag != TAG_START_DATABASE) { std::cout << "tag " << int(tag) << std::endl; std::cout << "tag " << tag << std::endl; } ASSERT(tag == TAG_START_DATABASE); if (tag == TAG_START_DATABASE) { name = readString(); id = readUnsigned(); count = readUnsigned(); } return true; // should only exit with (tag == TAG_START_DATABASE) } size_t Exporter::nextObject() { char tag = readTag(); if (tag == TAG_END_DATABASE) { unsigned long long objectCount = readUnsigned(); std::cout << "objectCount " << objectCount << " " << objectCount_ << std::endl; ASSERT(objectCount == objectCount_); return 0; } if (tag != TAG_START_OBJECT) { std::cout << tag << std::endl; } ASSERT(tag == TAG_START_OBJECT); objectCount_++; subCount_ = 0; stack_.clear(); try { type_ = readUnsigned(); location_ = readUnsigned(); objectId_ = readUnsigned(); return readUnsigned(); // count } catch (...) { std::cout << "ERROR reading start object " << *this << std::endl; throw; } } void _nextSubObject(eckit::Exporter& e) { e.nextSubObject(); } std::string Exporter::path() const { std::string s; for (std::vector::const_iterator j = stack_.begin(); j != stack_.end(); ++j) { if (s.length()) { s += "."; } s += (*j); } return s; } void Exporter::nextSubObject() { ASSERT(readTag() == TAG_START_SUBOBJECT); subCount_++; for (std::map::iterator j = members_.begin(); j != members_.end(); ++j) { if (!(*j).second.used()) { std::cout << "WARNING NOT USED [" << (*j).first << "]" << std::endl; } } members_.clear(); std::string s; for (;;) { char tag = readTag(); switch (tag) { case TAG_START_CLASS: s = readString(); // cout << s << std::endl; // stack_.push_back(s.substr(0,s.find('<'))); stack_.push_back(s); // stack_.push_back(s); break; case TAG_START_MEMBER: stack_.push_back(readString()); break; case TAG_END_MEMBER: stack_.pop_back(); break; case TAG_END_CLASS: stack_.pop_back(); break; case TAG_UNSIGNED: { std::string p = path(); Datatype& x = members_[p]; ASSERT(x.empty()); x = Datatype(_readUnsigned()); // cout << "read [" << p << "] = " << x << std::endl; } break; case TAG_SIGNED: { std::string p = path(); Datatype& x = members_[p]; ASSERT(x.empty()); x = Datatype(_readSigned()); // cout << "read [" << p << "] = " << x << std::endl; } break; case TAG_DOUBLE: { std::string p = path(); Datatype& x = members_[p]; ASSERT(x.empty()); x = Datatype(_readDouble()); // cout << "read [" << p << "] = " << x << std::endl; } break; case TAG_END_SUBOBJECT: return; default: std::cout << tag << std::endl; ASSERT(1 == 0); break; } } } double Exporter::getDoubleMember(const std::string& name) { std::map::iterator j = members_.find(name); if (j != members_.end()) { // cout << "consume [" << name << "] = " << (*j).second << std::endl; return (*j).second; } std::cout << name << " not found" << std::endl; return 0; } long long Exporter::getSignedMember(const std::string& name) { std::map::iterator j = members_.find(name); if (j != members_.end()) { // cout << "consume [" << name << "] = " << (*j).second << std::endl; return (*j).second; } std::cout << name << " not found" << std::endl; return 0; } unsigned long long Exporter::getUnsignedMember(const std::string& name) { std::map::iterator j = members_.find(name); if (j != members_.end()) { // cout << "consume [" << name << "] = " << (*j).second << std::endl; return (*j).second; } std::cout << name << " not found" << std::endl; return 0; } Evolve::Evolve(eckit::Exporter& e) : e_(e), parent_{nullptr} {} Evolve::Evolve(Evolve* e, char const* klass, char const* name) : e_(e->e_), path_(e->path()), parent_(e) { if (path_.length()) { path_ += "."; } path_ += klass; if (name) { path_ += "."; path_ += name; } } Evolve Evolve::operator()(char const* klass, char const* name) { return Evolve(this, klass, name); } Evolve::operator unsigned char() { return e_.getUnsignedMember(path_); } Evolve::operator unsigned int() { return e_.getUnsignedMember(path_); } Evolve::operator int() { return e_.getSignedMember(path_); } Evolve::operator char() { return e_.getSignedMember(path_); } Evolve::operator unsigned short() { return e_.getUnsignedMember(path_); } Evolve::operator unsigned long() { return e_.getUnsignedMember(path_); } Evolve::operator long long() { return e_.getSignedMember(path_); } Evolve::operator unsigned long long() { return e_.getUnsignedMember(path_); } Evolve::operator double() { return e_.getDoubleMember(path_); } Evolve::operator long() { return e_.getSignedMember(path_); } Exporter::Datatype::Datatype() : type_(0), used_(false), double_(0), signed_(0), unsigned_(0) {} Exporter::Datatype::Datatype(double d) : type_(TAG_DOUBLE), used_(false), double_(d), signed_(0), unsigned_(0) {} Exporter::Datatype::Datatype(long long d) : type_(TAG_SIGNED), used_(false), double_(0), signed_(d), unsigned_(0) {} Exporter::Datatype::Datatype(unsigned long long d) : type_(TAG_UNSIGNED), used_(false), double_(0), signed_(0), unsigned_(d) {} Exporter::Datatype::operator long long() { ASSERT(type_ == TAG_SIGNED); ASSERT(!used_); used_ = true; return signed_; } Exporter::Datatype::operator unsigned long long() { ASSERT(type_ == TAG_UNSIGNED); ASSERT(!used_); used_ = true; return unsigned_; } Exporter::Datatype::operator double() { ASSERT(type_ == TAG_DOUBLE); ASSERT(!used_); used_ = true; return double_; } void Exporter::Datatype::print(std::ostream& out) const { switch (type_) { case TAG_SIGNED: out << "S(" << signed_; break; case TAG_UNSIGNED: out << "U(" << unsigned_; break; case TAG_DOUBLE: out << "D(" << double_; break; default: out << "X("; break; } out << "," << (used_ ? "used" : "new") << ")"; } #define X(a) out << " " << #a << "=" << a; void Exporter::print(std::ostream& out) const { out << "Exporter["; X(objectCount_); X(subCount_); X(type_); X(location_); X(objectId_); X(inObject_); X(path()); out << "]"; } void Exporter::openForWrite() { handle().openForWrite(0); } void Exporter::close() { writeTag(TAG_EOF); handle().close(); } void Exporter::startSchemas() { writeTag(TAG_START_SCHEMAS); } void Exporter::endSchemas() { writeTag(TAG_END_SCHEMAS); } void Exporter::startDatabase(const std::string& path, unsigned long id, unsigned long long count) { PathName home("~"); std::string p = path; if (p.find(home) == 0) { p = std::string("~/") + p.substr(std::string(home).length()); } writeTag(TAG_START_DATABASE); writeString(p); writeUnsigned(id); writeUnsigned(count); } void Exporter::endDatabase(const std::string&, unsigned long) { writeTag(TAG_END_DATABASE); writeUnsigned(objectCount_); objectCount_ = 0; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/linalg/0000775000175000017500000000000015161702250016136 5ustar alastairalastaireckit-2.0.7/src/eckit/linalg/LinearAlgebraSparse.cc0000664000175000017500000000402315161702250022312 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/LinearAlgebraSparse.h" #include "eckit/eckit.h" #include "eckit/linalg/BackendRegistry.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" namespace eckit::linalg { //----------------------------------------------------------------------------- static pthread_once_t once = PTHREAD_ONCE_INIT; static BackendRegistry* backends = nullptr; static void init() { backends = new BackendRegistry( #if eckit_HAVE_EIGEN "eigen" #else "generic" #endif , "ECKIT_LINEAR_ALGEBRA_SPARSE_BACKEND"); } //----------------------------------------------------------------------------- const LinearAlgebraSparse& LinearAlgebraSparse::backend(const std::string& name) { pthread_once(&once, init); if (!name.empty()) { backends->backend(name); } return backends->find(); } const LinearAlgebraSparse& LinearAlgebraSparse::getBackend(const std::string& name) { pthread_once(&once, init); return backends->find(name); } bool LinearAlgebraSparse::hasBackend(const std::string& name) { pthread_once(&once, init); return backends->has(name); } std::ostream& LinearAlgebraSparse::list(std::ostream& out) { pthread_once(&once, init); return backends->list(out); } const std::string& LinearAlgebraSparse::name() { return backends->name(); } LinearAlgebraSparse::LinearAlgebraSparse(const std::string& name) { pthread_once(&once, init); backends->add(name, this); } //----------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/LinearAlgebraDense.cc0000664000175000017500000000412115161702250022112 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/LinearAlgebraDense.h" #include "eckit/eckit.h" #include "eckit/linalg/BackendRegistry.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" namespace eckit::linalg { //----------------------------------------------------------------------------- static pthread_once_t once = PTHREAD_ONCE_INIT; static BackendRegistry* backends = nullptr; static void init() { backends = new BackendRegistry( #if eckit_HAVE_MKL "mkl" #elif eckit_HAVE_LAPACK "lapack" #elif eckit_HAVE_EIGEN "eigen" #else "generic" #endif , "ECKIT_LINEAR_ALGEBRA_DENSE_BACKEND"); } //----------------------------------------------------------------------------- const LinearAlgebraDense& LinearAlgebraDense::backend(const std::string& name) { pthread_once(&once, init); if (!name.empty()) { backends->backend(name); } return backends->find(); } const LinearAlgebraDense& LinearAlgebraDense::getBackend(const std::string& name) { pthread_once(&once, init); return backends->find(name); } bool LinearAlgebraDense::hasBackend(const std::string& name) { pthread_once(&once, init); return backends->has(name); } std::ostream& LinearAlgebraDense::list(std::ostream& out) { pthread_once(&once, init); return backends->list(out); } const std::string& LinearAlgebraDense::name() { return backends->name(); } LinearAlgebraDense::LinearAlgebraDense(const std::string& name) { pthread_once(&once, init); backends->add(name, this); } //----------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/Tensor.cc0000664000175000017500000000115415161702250017720 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/Tensor.h" namespace eckit::linalg { // Explicit template instantiation to minimise dynamic library code bloat template class Tensor; template class Tensor; } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/SparseMatrix.h0000664000175000017500000002150615161702250020735 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ /// @author Florian Rathgeber /// @author Pedro Maciel /// @author Tiago Quintino #pragma once #include #include #include #include #include "eckit/linalg/Triplet.h" #include "eckit/linalg/types.h" namespace eckit { class MemoryBuffer; class PathName; class Stream; } // namespace eckit namespace eckit::linalg { //---------------------------------------------------------------------------------------------------------------------- /// Sparse matrix in CRS (compressed row storage) format class SparseMatrix { public: using UIndex = std::make_unsigned_t; struct Layout { Layout() = default; void reset() { data_ = nullptr; outer_ = nullptr; inner_ = nullptr; } Scalar* data_ = nullptr; ///< matrix entries, sized with number of non-zeros (nnz) UIndex* outer_ = nullptr; ///< start of rows, sized number of rows + 1 Index* inner_ = nullptr; ///< column indices, sized with number of non-zeros (nnz) }; struct Shape { Shape() = default; void reset() { size_ = 0; rows_ = 0; cols_ = 0; } /// @returns number of rows Size rows() const { return rows_; } /// @returns number of columns Size cols() const { return cols_; } /// @returns number of non-zeros Size nonZeros() const { return size_; } /// @returns number of non-zeros Size dataSize() const { return nonZeros(); } /// @returns number of non-zeros Size innerSize() const { return nonZeros(); } /// @returns outer size is number of rows + 1 Size outerSize() const { return rows_ + 1; } size_t allocSize() const { return sizeofData() + sizeofOuter() + sizeofInner(); } size_t sizeofData() const { return dataSize() * sizeof(Scalar); } size_t sizeofOuter() const { return outerSize() * sizeof(UIndex); } size_t sizeofInner() const { return innerSize() * sizeof(Index); } Size size_ = 0; ///< Size of the container (AKA number of non-zeros nnz) Size rows_ = 0; ///< Number of rows Size cols_ = 0; ///< Number of columns void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& os, const Shape& p) { p.print(os); return os; } }; class Allocator { public: virtual ~Allocator(); /// @note that shape may be modified by the allocator, e.g. loading of pre-computed matrices virtual Layout allocate(Shape&) = 0; virtual void deallocate(Layout, Shape) = 0; /// @returns if allocation is in shared memory virtual bool inSharedMemory() const = 0; virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream& os, const Allocator& a) { a.print(os); return os; } }; public: // -- Constructors /// Default constructor, empty matrix explicit SparseMatrix(Allocator* = nullptr); /// Constructs an identity matrix with provided dimensions SparseMatrix(Size rows, Size cols, Allocator* = nullptr); /// Constructor from triplets SparseMatrix(Size rows, Size cols, const std::vector&); /// Constructor from Stream explicit SparseMatrix(Stream&); /// Constructor from MemoryBuffer explicit SparseMatrix(const MemoryBuffer&); /// Move constructor SparseMatrix(SparseMatrix&&); /// Copy constructor SparseMatrix(const SparseMatrix&); /// Destructor ~SparseMatrix(); /// Assignment operator (allocates and copies data) SparseMatrix& operator=(const SparseMatrix&); /// Assignment operator (moves data) SparseMatrix& operator=(SparseMatrix&&); public: /// Prune entries with exactly the given value SparseMatrix& prune(Scalar = 0); /// Set matrix to the identity SparseMatrix& setIdentity(Size rows, Size cols); /// Transpose matrix in-place SparseMatrix& transpose(); /// @returns a sparse matrix that is a row reduction and reorder accoring to indexes passed in vector SparseMatrix rowReduction(const std::vector& p) const; // -- I/O void save(const eckit::PathName&) const; void load(const eckit::PathName&); void dump(eckit::MemoryBuffer&) const; void dump(void* buffer, size_t size) const; static void load(const void* buffer, size_t bufferSize, Layout&, Shape&); ///< from dump() void swap(SparseMatrix&); /// @returns number of rows Size rows() const { return shape_.rows_; } /// @returns number of columns Size cols() const { return shape_.cols_; } /// @returns number of non-zeros Size nonZeros() const { return shape_.size_; } /// @returns true if this matrix does not contain non-zero entries bool empty() const { return nonZeros() == 0; } /// @returns read-only view of the outer index vector const UIndex* outerIndex() const { return spm_.outer_; } /// @returns read-only view of the data vector const Scalar* data() const { return spm_.data_; } /// @returns read-only view of the outer index vector (signed) const Index* outer() const; /// @returns read-only view of the inner index vector const Index* inner() const { return spm_.inner_; } /// Set a new number of columns (may invalidate data) void cols(Size cols); /// Reserve memory for given number of non-zeros (invalidates all data arrays) void reserve(Size rows, Size cols, Size nnz); /// @returns footprint of the matrix in memory size_t footprint() const; /// @returns if allocation is in shared memory bool inSharedMemory() const; void dump(std::ostream&) const; void print(std::ostream&) const; const Allocator& owner() const; friend std::ostream& operator<<(std::ostream& os, const SparseMatrix& m) { m.print(os); return os; } public: // iterators struct iterator; struct const_iterator { const_iterator(const SparseMatrix&); const_iterator(const SparseMatrix&, Size row); const_iterator(const const_iterator&) = default; const_iterator(const_iterator&&) = default; virtual ~const_iterator() = default; Size col() const; Size row() const; operator bool() const { return matrix_ && (index_ < matrix_->nonZeros()); } const_iterator& operator++(); const_iterator operator++(int); const_iterator& operator=(const const_iterator&) = default; const_iterator& operator=(const_iterator&&) = default; bool operator!=(const const_iterator& other) const { return !operator==(other); } bool operator==(const const_iterator&) const; const Scalar& operator*() const; void print(std::ostream&) const; bool lastOfRow() const { return ((index_ + 1) == static_cast(matrix_->spm_.outer_[row_ + 1])); } private: friend struct iterator; SparseMatrix* matrix_; Size index_; Size row_; }; struct iterator final : const_iterator { using const_iterator::const_iterator; Scalar& operator*(); }; /// const iterators to begin/end of row const_iterator begin(Size row) const { return {*this, row}; } const_iterator end(Size row) const { return {*this, row + 1}; } /// const iterators to begin/end of matrix const_iterator begin() const { return {*this}; } const_iterator end() const { return {*this, rows()}; } /// iterators to begin/end of row iterator begin(Size row) { return {*this, row}; } iterator end(Size row) { return {*this, row + 1}; } /// const iterators to begin/end of matrix iterator begin() { return {*this}; } iterator end() { return {*this, rows()}; } private: /// Resets the matrix to a deallocated state void reset(); /// Serialise to a Stream void encode(Stream&) const; /// Deserialise from a Stream void decode(Stream&); private: Layout spm_; ///< Matrix layout Shape shape_; ///< Matrix shape std::unique_ptr owner_; ///< Matrix memory manager/allocator friend Stream& operator<<(Stream&, const SparseMatrix&); }; Stream& operator<<(Stream&, const SparseMatrix&); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/LinearAlgebra.cc0000664000175000017500000000431415161702250021137 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/LinearAlgebra.h" #include #include #include "eckit/exception/Exceptions.h" #include "eckit/utils/StringTools.h" namespace eckit::linalg { //----------------------------------------------------------------------------- std::ostream& LinearAlgebra::list(std::ostream& out) { const auto* delimiter = ", "; const auto* sep = ""; std::ostringstream str; LinearAlgebraDense::list(str); LinearAlgebraSparse::list(str.str().empty() ? str : (str << delimiter)); for (const auto& backend : [](const std::vector& unsorted) { return std::set(unsorted.begin(), unsorted.end()); }(StringTools::split(delimiter, str.str()))) { out << sep << backend; sep = delimiter; } return out; } bool LinearAlgebra::hasBackend(const std::string& name) { return LinearAlgebraDense::hasBackend(name) || LinearAlgebraSparse::hasBackend(name); } const LinearAlgebraDense& LinearAlgebra::getDenseBackend(const std::string& name) { return LinearAlgebraDense::getBackend(name); } const LinearAlgebraSparse& LinearAlgebra::getSparseBackend(const std::string& name) { return LinearAlgebraSparse::getBackend(name); } const LinearAlgebraDense& LinearAlgebra::denseBackend(const std::string& name) { return LinearAlgebraDense::backend(name); } const LinearAlgebraSparse& LinearAlgebra::sparseBackend(const std::string& name) { return LinearAlgebraSparse::backend(name); } bool LinearAlgebra::hasDenseBackend(const std::string& name) { return LinearAlgebraDense::hasBackend(name); } bool LinearAlgebra::hasSparseBackend(const std::string& name) { return LinearAlgebraSparse::hasBackend(name); } //----------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/sparse/0000775000175000017500000000000015161702250017433 5ustar alastairalastaireckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraGeneric.h0000664000175000017500000000175215161702250023576 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraSparse.h" namespace eckit::linalg::sparse { struct LinearAlgebraGeneric final : public LinearAlgebraSparse { LinearAlgebraGeneric() {} LinearAlgebraGeneric(const std::string& name) : LinearAlgebraSparse(name) {} void spmv(const SparseMatrix&, const Vector&, Vector&) const override; void spmm(const SparseMatrix&, const Matrix&, Matrix&) const override; void dsptd(const Vector&, const SparseMatrix&, const Vector&, SparseMatrix&) const override; void print(std::ostream&) const override; }; } // namespace eckit::linalg::sparse eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraEigen.cc0000664000175000017500000000472515161702250023412 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/sparse/LinearAlgebraEigen.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/SparseMatrix.h" #include "eckit/linalg/Vector.h" #include "eckit/linalg/sparse/LinearAlgebraGeneric.h" #include "eckit/maths/Eigen.h" namespace eckit::linalg::sparse { static const LinearAlgebraEigen __la("eigen"); using vec_t = Eigen::VectorXd::MapType; using mat_t = Eigen::MatrixXd::MapType; using spm_t = Eigen::Map>; void LinearAlgebraEigen::print(std::ostream& out) const { out << "LinearAlgebraEigen[]"; } void LinearAlgebraEigen::spmv(const SparseMatrix& A, const Vector& x, Vector& y) const { ASSERT(x.size() == A.cols()); ASSERT(y.size() == A.rows()); // We expect indices to be 0-based ASSERT(A.outer()[0] == 0); // Eigen requires non-const pointers to the data spm_t Ai(A.rows(), A.cols(), A.nonZeros(), const_cast(A.outer()), const_cast(A.inner()), const_cast(A.data())); vec_t xi(Eigen::VectorXd::Map(const_cast(x.data()), x.size())); vec_t yi(Eigen::VectorXd::Map(y.data(), y.size())); yi = Ai * xi; } void LinearAlgebraEigen::spmm(const SparseMatrix& A, const Matrix& B, Matrix& C) const { ASSERT(A.cols() == B.rows()); ASSERT(A.rows() == C.rows()); ASSERT(B.cols() == C.cols()); // We expect indices to be 0-based ASSERT(A.outer()[0] == 0); // Eigen requires non-const pointers to the data spm_t Ai(A.rows(), A.cols(), A.nonZeros(), const_cast(A.outer()), const_cast(A.inner()), const_cast(A.data())); mat_t Bi(Eigen::MatrixXd::Map(const_cast(B.data()), B.rows(), B.cols())); mat_t Ci(Eigen::MatrixXd::Map(C.data(), C.rows(), C.cols())); Ci = Ai * Bi; } void LinearAlgebraEigen::dsptd(const Vector& x, const SparseMatrix& A, const Vector& y, SparseMatrix& B) const { static const sparse::LinearAlgebraGeneric generic; generic.dsptd(x, A, y, B); } } // namespace eckit::linalg::sparse eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraMKL.h0000664000175000017500000000202515161702250022637 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraSparse.h" namespace eckit { namespace linalg { namespace sparse { struct LinearAlgebraMKL final : public LinearAlgebraSparse { LinearAlgebraMKL() {} LinearAlgebraMKL(const std::string& name) : LinearAlgebraSparse(name) {} void spmv(const SparseMatrix&, const Vector&, Vector&) const override; void spmm(const SparseMatrix&, const Matrix&, Matrix&) const override; void dsptd(const Vector&, const SparseMatrix&, const Vector&, SparseMatrix&) const override; void print(std::ostream&) const override; }; } // namespace sparse } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraCUDA.h0000664000175000017500000000203015161702250022724 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraSparse.h" namespace eckit { namespace linalg { namespace sparse { struct LinearAlgebraCUDA final : public LinearAlgebraSparse { LinearAlgebraCUDA() {} LinearAlgebraCUDA(const std::string& name) : LinearAlgebraSparse(name) {} void spmv(const SparseMatrix&, const Vector&, Vector&) const override; void spmm(const SparseMatrix&, const Matrix&, Matrix&) const override; void dsptd(const Vector&, const SparseMatrix&, const Vector&, SparseMatrix&) const override; void print(std::ostream&) const override; }; } // namespace sparse } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraEigen.h0000664000175000017500000000174415161702250023252 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraSparse.h" namespace eckit::linalg::sparse { struct LinearAlgebraEigen final : public LinearAlgebraSparse { LinearAlgebraEigen() {} LinearAlgebraEigen(const std::string& name) : LinearAlgebraSparse(name) {} void spmv(const SparseMatrix&, const Vector&, Vector&) const override; void spmm(const SparseMatrix&, const Matrix&, Matrix&) const override; void dsptd(const Vector&, const SparseMatrix&, const Vector&, SparseMatrix&) const override; void print(std::ostream&) const override; }; } // namespace eckit::linalg::sparse eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraHIP.cc0000664000175000017500000001744215161702250023003 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/sparse/LinearAlgebraHIP.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/SparseMatrix.h" #include "eckit/linalg/Vector.h" #include "eckit/linalg/detail/HIP.h" #include "eckit/linalg/sparse/LinearAlgebraGeneric.h" namespace eckit { namespace linalg { namespace sparse { static const LinearAlgebraHIP __la("hip"); void LinearAlgebraHIP::print(std::ostream& out) const { out << "LinearAlgebraHIP[]"; } void LinearAlgebraHIP::spmv(const SparseMatrix& A, const Vector& x, Vector& y) const { ASSERT(x.size() == A.cols() && y.size() == A.rows()); // We expect indices to be 0-based ASSERT(A.outer()[0] == 0); const Size sizeArowptr = (A.rows() + 1) * sizeof(Index); const Size sizeAcolidx = A.nonZeros() * sizeof(Index); const Size sizeAvalues = A.nonZeros() * sizeof(Scalar); const Size sizex = A.cols() * sizeof(Scalar); const Size sizey = A.rows() * sizeof(Scalar); Index* d_A_rowptr; ///< device memory matrix A row pointers Index* d_A_colidx; ///< device memory matrix A col indices Scalar* d_A_values; ///< device memory matrix A values Scalar* d_x; ///< device memory vector x Scalar* d_y; ///< device memory vector y CALL_HIP(hipMalloc((void**)&d_A_rowptr, sizeArowptr)); CALL_HIP(hipMalloc((void**)&d_A_colidx, sizeAcolidx)); CALL_HIP(hipMalloc((void**)&d_A_values, sizeAvalues)); CALL_HIP(hipMalloc((void**)&d_x, sizex)); CALL_HIP(hipMalloc((void**)&d_y, sizey)); CALL_HIP(hipMemcpy(d_A_rowptr, A.outer(), sizeArowptr, hipMemcpyHostToDevice)); CALL_HIP(hipMemcpy(d_A_colidx, A.inner(), sizeAcolidx, hipMemcpyHostToDevice)); CALL_HIP(hipMemcpy(d_A_values, A.data(), sizeAvalues, hipMemcpyHostToDevice)); CALL_HIP(hipMemcpy(d_x, x.data(), sizex, hipMemcpyHostToDevice)); hipsparseHandle_t handle; CALL_HIPSPARSE(hipsparseCreate(&handle)); hipsparseSpMatDescr_t matA; CALL_HIPSPARSE(hipsparseCreateCsr(&matA, A.rows(), A.cols(), A.nonZeros(), d_A_rowptr, d_A_colidx, d_A_values, HIPSPARSE_INDEX_32I, HIPSPARSE_INDEX_32I, HIPSPARSE_INDEX_BASE_ZERO, HIP_R_64F)); hipsparseDnVecDescr_t vecX; CALL_HIPSPARSE(hipsparseCreateDnVec(&vecX, x.size(), d_x, HIP_R_64F)); hipsparseDnVecDescr_t vecY; CALL_HIPSPARSE(hipsparseCreateDnVec(&vecY, y.size(), d_y, HIP_R_64F)); const Scalar alpha = 1.0; const Scalar beta = 0.0; // Determine buffer size size_t bufferSize = 0; CALL_HIPSPARSE(hipsparseSpMV_bufferSize(handle, HIPSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY, HIP_R_64F, HIPSPARSE_SPMV_ALG_DEFAULT, &bufferSize)); // Allocate buffer char* buffer; CALL_HIP(hipMalloc(&buffer, bufferSize)); // Perform SpMV // y = alpha * A * x + beta * y CALL_HIPSPARSE(hipsparseSpMV(handle, HIPSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY, HIP_R_64F, HIPSPARSE_SPMV_ALG_DEFAULT, buffer)); // Copy result back to host CALL_HIP(hipMemcpy(y.data(), d_y, sizey, hipMemcpyDeviceToHost)); CALL_HIPSPARSE(hipsparseDestroyDnVec(vecY)); CALL_HIPSPARSE(hipsparseDestroyDnVec(vecX)); CALL_HIPSPARSE(hipsparseDestroySpMat(matA)); CALL_HIPSPARSE(hipsparseDestroy(handle)); CALL_HIP(hipFree(d_A_rowptr)); CALL_HIP(hipFree(d_A_colidx)); CALL_HIP(hipFree(d_A_values)); CALL_HIP(hipFree(d_x)); CALL_HIP(hipFree(d_y)); } void LinearAlgebraHIP::spmm(const SparseMatrix& A, const Matrix& B, Matrix& C) const { ASSERT(A.cols() == B.rows() && A.rows() == C.rows() && B.cols() == C.cols()); // We expect indices to be 0-based ASSERT(A.outer()[0] == 0); const Size sizeArowptr = (A.rows() + 1) * sizeof(Index); const Size sizeAcolidx = A.nonZeros() * sizeof(Index); const Size sizeAvalues = A.nonZeros() * sizeof(Scalar); const Size sizeB = B.rows() * B.cols() * sizeof(Scalar); const Size sizeC = A.rows() * B.cols() * sizeof(Scalar); Index* d_A_rowptr; ///< device memory matrix A row pointers Index* d_A_colidx; ///< device memory matrix A col indices Scalar* d_A_values; ///< device memory matrix A values Scalar* d_B; ///< device memory matrix B Scalar* d_C; ///< device memory matrix C CALL_HIP(hipMalloc((void**)&d_A_rowptr, sizeArowptr)); CALL_HIP(hipMalloc((void**)&d_A_colidx, sizeAcolidx)); CALL_HIP(hipMalloc((void**)&d_A_values, sizeAvalues)); CALL_HIP(hipMalloc((void**)&d_B, sizeB)); CALL_HIP(hipMalloc((void**)&d_C, sizeC)); CALL_HIP(hipMemcpy(d_A_rowptr, A.outer(), sizeArowptr, hipMemcpyHostToDevice)); CALL_HIP(hipMemcpy(d_A_colidx, A.inner(), sizeAcolidx, hipMemcpyHostToDevice)); CALL_HIP(hipMemcpy(d_A_values, A.data(), sizeAvalues, hipMemcpyHostToDevice)); CALL_HIP(hipMemcpy(d_B, B.data(), sizeB, hipMemcpyHostToDevice)); hipsparseHandle_t handle; CALL_HIPSPARSE(hipsparseCreate(&handle)); hipsparseSpMatDescr_t matA; CALL_HIPSPARSE(hipsparseCreateCsr(&matA, A.rows(), A.cols(), A.nonZeros(), d_A_rowptr, d_A_colidx, d_A_values, HIPSPARSE_INDEX_32I, HIPSPARSE_INDEX_32I, HIPSPARSE_INDEX_BASE_ZERO, HIP_R_64F)); // Create dense matrix descriptors hipsparseDnMatDescr_t matB; CALL_HIPSPARSE(hipsparseCreateDnMat(&matB, B.rows(), // rows B.cols(), // cols B.rows(), // leading dimension d_B, HIP_R_64F, HIPSPARSE_ORDER_COL)); hipsparseDnMatDescr_t matC; CALL_HIPSPARSE(hipsparseCreateDnMat(&matC, C.rows(), // rows C.cols(), // cols C.rows(), // leading dimension d_C, HIP_R_64F, HIPSPARSE_ORDER_COL)); const Scalar alpha = 1.0; const Scalar beta = 0.0; size_t bufferSize = 0; CALL_HIPSPARSE(hipsparseSpMM_bufferSize(handle, HIPSPARSE_OPERATION_NON_TRANSPOSE, HIPSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, matB, &beta, matC, HIP_R_64F, HIPSPARSE_SPMM_ALG_DEFAULT, &bufferSize)); // Allocate buffer char* buffer; CALL_HIP(hipMalloc(&buffer, bufferSize)); // Perform SpMM CALL_HIPSPARSE(hipsparseSpMM(handle, HIPSPARSE_OPERATION_NON_TRANSPOSE, HIPSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, matB, &beta, matC, HIP_R_64F, HIPSPARSE_SPMM_ALG_DEFAULT, buffer)); CALL_HIP(hipMemcpy(C.data(), d_C, sizeC, hipMemcpyDeviceToHost)); CALL_HIPSPARSE(hipsparseDestroy(handle)); CALL_HIPSPARSE(hipsparseDestroyDnMat(matC)); CALL_HIPSPARSE(hipsparseDestroyDnMat(matB)); CALL_HIPSPARSE(hipsparseDestroySpMat(matA)); CALL_HIP(hipFree(buffer)); CALL_HIP(hipFree(d_A_rowptr)); CALL_HIP(hipFree(d_A_colidx)); CALL_HIP(hipFree(d_A_values)); CALL_HIP(hipFree(d_B)); CALL_HIP(hipFree(d_C)); } void LinearAlgebraHIP::dsptd(const Vector& x, const SparseMatrix& A, const Vector& y, SparseMatrix& B) const { static const sparse::LinearAlgebraGeneric generic; generic.dsptd(x, A, y, B); } } // namespace sparse } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraTorch.cc0000664000175000017500000000544615161702250023443 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/sparse/LinearAlgebraTorch.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/SparseMatrix.h" #include "eckit/linalg/Vector.h" #include "eckit/linalg/detail/Torch.h" #include "eckit/linalg/sparse/LinearAlgebraGeneric.h" namespace eckit::linalg::sparse { static const LinearAlgebraTorch LA_TORCH_CPU_1("torch", torch::DeviceType::CPU); static const LinearAlgebraTorch LA_TORCH_CPU_2("torch-cpu", torch::DeviceType::CPU); static const LinearAlgebraTorch LA_TORCH_CUDA("torch-cuda", torch::DeviceType::CUDA); static const LinearAlgebraTorch LA_TORCH_HIP("torch-hip", torch::DeviceType::HIP); // static const LinearAlgebraTorch LA_TORCH_MPS("torch-mps", torch::DeviceType::MPS); static const LinearAlgebraTorch LA_TORCH_XPU("torch-xpu", torch::DeviceType::XPU); static const LinearAlgebraTorch LA_TORCH_XLA("torch-xla", torch::DeviceType::XLA); static const LinearAlgebraTorch LA_TORCH_META("torch-meta", torch::DeviceType::Meta); void LinearAlgebraTorch::spmv(const SparseMatrix& A, const Vector& x, Vector& y) const { auto Ni = static_cast(A.rows()); auto Nj = static_cast(A.cols()); ASSERT(Ni == y.rows()); ASSERT(Nj == x.rows()); // multiplication auto A_tensor = make_sparse_csr_tensor(A); auto x_tensor = make_dense_tensor(x); auto y_tensor = tensor_to_host(torch::matmul(A_tensor, x_tensor)); // assignment std::memcpy(y.data(), y_tensor.data_ptr(), Ni * sizeof(Scalar)); } void LinearAlgebraTorch::spmm(const SparseMatrix& A, const Matrix& X, Matrix& Y) const { auto Ni = static_cast(A.rows()); auto Nj = static_cast(A.cols()); auto Nk = static_cast(X.cols()); ASSERT(Ni == Y.rows()); ASSERT(Nj == X.rows()); ASSERT(Nk == Y.cols()); // multiplication and conversion from column-major to row-major (and back) auto A_tensor = make_sparse_csr_tensor(A); auto X_tensor = make_dense_tensor(X); auto Y_tensor = tensor_transpose(tensor_to_host(torch::matmul(A_tensor, X_tensor))); // assignment std::memcpy(Y.data(), Y_tensor.data_ptr(), Y.size() * sizeof(Scalar)); } void LinearAlgebraTorch::dsptd(const Vector& x, const SparseMatrix& A, const Vector& y, SparseMatrix& B) const { static const sparse::LinearAlgebraGeneric generic; generic.dsptd(x, A, y, B); } } // namespace eckit::linalg::sparse eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraHIP.h0000664000175000017500000000202515161702250022634 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraSparse.h" namespace eckit { namespace linalg { namespace sparse { struct LinearAlgebraHIP final : public LinearAlgebraSparse { LinearAlgebraHIP() {} LinearAlgebraHIP(const std::string& name) : LinearAlgebraSparse(name) {} void spmv(const SparseMatrix&, const Vector&, Vector&) const override; void spmm(const SparseMatrix&, const Matrix&, Matrix&) const override; void dsptd(const Vector&, const SparseMatrix&, const Vector&, SparseMatrix&) const override; void print(std::ostream&) const override; }; } // namespace sparse } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraCUDA.cc0000664000175000017500000001744115161702250023076 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/sparse/LinearAlgebraCUDA.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/SparseMatrix.h" #include "eckit/linalg/Vector.h" #include "eckit/linalg/detail/CUDA.h" #include "eckit/linalg/sparse/LinearAlgebraGeneric.h" namespace eckit { namespace linalg { namespace sparse { static const LinearAlgebraCUDA __la("cuda"); void LinearAlgebraCUDA::print(std::ostream& out) const { out << "LinearAlgebraCUDA[]"; } void LinearAlgebraCUDA::spmv(const SparseMatrix& A, const Vector& x, Vector& y) const { ASSERT(x.size() == A.cols() && y.size() == A.rows()); // We expect indices to be 0-based ASSERT(A.outer()[0] == 0); const Size sizeArowptr = (A.rows() + 1) * sizeof(Index); const Size sizeAcolidx = A.nonZeros() * sizeof(Index); const Size sizeAvalues = A.nonZeros() * sizeof(Scalar); const Size sizex = A.cols() * sizeof(Scalar); const Size sizey = A.rows() * sizeof(Scalar); Index* d_A_rowptr; ///< device memory matrix A row pointers Index* d_A_colidx; ///< device memory matrix A col indices Scalar* d_A_values; ///< device memory matrix A values Scalar* d_x; ///< device memory vector x Scalar* d_y; ///< device memory vector y CALL_CUDA(cudaMalloc((void**)&d_A_rowptr, sizeArowptr)); CALL_CUDA(cudaMalloc((void**)&d_A_colidx, sizeAcolidx)); CALL_CUDA(cudaMalloc((void**)&d_A_values, sizeAvalues)); CALL_CUDA(cudaMalloc((void**)&d_x, sizex)); CALL_CUDA(cudaMalloc((void**)&d_y, sizey)); CALL_CUDA(cudaMemcpy(d_A_rowptr, A.outer(), sizeArowptr, cudaMemcpyHostToDevice)); CALL_CUDA(cudaMemcpy(d_A_colidx, A.inner(), sizeAcolidx, cudaMemcpyHostToDevice)); CALL_CUDA(cudaMemcpy(d_A_values, A.data(), sizeAvalues, cudaMemcpyHostToDevice)); CALL_CUDA(cudaMemcpy(d_x, x.data(), sizex, cudaMemcpyHostToDevice)); cusparseHandle_t handle; CALL_CUSPARSE(cusparseCreate(&handle)); cusparseSpMatDescr_t matA; CALL_CUSPARSE(cusparseCreateCsr(&matA, A.rows(), A.cols(), A.nonZeros(), d_A_rowptr, d_A_colidx, d_A_values, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F)); cusparseDnVecDescr_t vecX; CALL_CUSPARSE(cusparseCreateDnVec(&vecX, x.size(), d_x, CUDA_R_64F)); cusparseDnVecDescr_t vecY; CALL_CUSPARSE(cusparseCreateDnVec(&vecY, y.size(), d_y, CUDA_R_64F)); const Scalar alpha = 1.0; const Scalar beta = 0.0; // Determine buffer size size_t bufferSize = 0; CALL_CUSPARSE(cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY, CUDA_R_64F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize)); // Allocate buffer char* buffer; CALL_CUDA(cudaMalloc(&buffer, bufferSize)); // Perform SpMV // y = alpha * A * x + beta * y CALL_CUSPARSE(cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, vecX, &beta, vecY, CUDA_R_64F, CUSPARSE_SPMV_ALG_DEFAULT, buffer)); // Copy result back to host CALL_CUDA(cudaMemcpy(y.data(), d_y, sizey, cudaMemcpyDeviceToHost)); CALL_CUSPARSE(cusparseDestroyDnVec(vecY)); CALL_CUSPARSE(cusparseDestroyDnVec(vecX)); CALL_CUSPARSE(cusparseDestroySpMat(matA)); CALL_CUSPARSE(cusparseDestroy(handle)); CALL_CUDA(cudaFree(d_A_rowptr)); CALL_CUDA(cudaFree(d_A_colidx)); CALL_CUDA(cudaFree(d_A_values)); CALL_CUDA(cudaFree(d_x)); CALL_CUDA(cudaFree(d_y)); } void LinearAlgebraCUDA::spmm(const SparseMatrix& A, const Matrix& B, Matrix& C) const { ASSERT(A.cols() == B.rows() && A.rows() == C.rows() && B.cols() == C.cols()); // We expect indices to be 0-based ASSERT(A.outer()[0] == 0); const Size sizeArowptr = (A.rows() + 1) * sizeof(Index); const Size sizeAcolidx = A.nonZeros() * sizeof(Index); const Size sizeAvalues = A.nonZeros() * sizeof(Scalar); const Size sizeB = B.rows() * B.cols() * sizeof(Scalar); const Size sizeC = A.rows() * B.cols() * sizeof(Scalar); Index* d_A_rowptr; ///< device memory matrix A row pointers Index* d_A_colidx; ///< device memory matrix A col indices Scalar* d_A_values; ///< device memory matrix A values Scalar* d_B; ///< device memory matrix B Scalar* d_C; ///< device memory matrix C CALL_CUDA(cudaMalloc((void**)&d_A_rowptr, sizeArowptr)); CALL_CUDA(cudaMalloc((void**)&d_A_colidx, sizeAcolidx)); CALL_CUDA(cudaMalloc((void**)&d_A_values, sizeAvalues)); CALL_CUDA(cudaMalloc((void**)&d_B, sizeB)); CALL_CUDA(cudaMalloc((void**)&d_C, sizeC)); CALL_CUDA(cudaMemcpy(d_A_rowptr, A.outer(), sizeArowptr, cudaMemcpyHostToDevice)); CALL_CUDA(cudaMemcpy(d_A_colidx, A.inner(), sizeAcolidx, cudaMemcpyHostToDevice)); CALL_CUDA(cudaMemcpy(d_A_values, A.data(), sizeAvalues, cudaMemcpyHostToDevice)); CALL_CUDA(cudaMemcpy(d_B, B.data(), sizeB, cudaMemcpyHostToDevice)); cusparseHandle_t handle; CALL_CUSPARSE(cusparseCreate(&handle)); cusparseSpMatDescr_t matA; CALL_CUSPARSE(cusparseCreateCsr(&matA, A.rows(), A.cols(), A.nonZeros(), d_A_rowptr, d_A_colidx, d_A_values, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F)); // Create dense matrix descriptors cusparseDnMatDescr_t matB; CALL_CUSPARSE(cusparseCreateDnMat(&matB, B.rows(), // rows B.cols(), // cols B.rows(), // leading dimension d_B, CUDA_R_64F, CUSPARSE_ORDER_COL)); cusparseDnMatDescr_t matC; CALL_CUSPARSE(cusparseCreateDnMat(&matC, C.rows(), // rows C.cols(), // cols C.rows(), // leading dimension d_C, CUDA_R_64F, CUSPARSE_ORDER_COL)); const Scalar alpha = 1.0; const Scalar beta = 0.0; size_t bufferSize = 0; CALL_CUSPARSE(cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, matB, &beta, matC, CUDA_R_64F, CUSPARSE_SPMM_ALG_DEFAULT, &bufferSize)); // Allocate buffer char* buffer; CALL_CUDA(cudaMalloc(&buffer, bufferSize)); // Perform SpMM CALL_CUSPARSE(cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA, matB, &beta, matC, CUDA_R_64F, CUSPARSE_SPMM_ALG_DEFAULT, buffer)); CALL_CUDA(cudaMemcpy(C.data(), d_C, sizeC, cudaMemcpyDeviceToHost)); CALL_CUSPARSE(cusparseDestroy(handle)); CALL_CUSPARSE(cusparseDestroyDnMat(matC)); CALL_CUSPARSE(cusparseDestroyDnMat(matB)); CALL_CUSPARSE(cusparseDestroySpMat(matA)); CALL_CUDA(cudaFree(buffer)); CALL_CUDA(cudaFree(d_A_rowptr)); CALL_CUDA(cudaFree(d_A_colidx)); CALL_CUDA(cudaFree(d_A_values)); CALL_CUDA(cudaFree(d_B)); CALL_CUDA(cudaFree(d_C)); } void LinearAlgebraCUDA::dsptd(const Vector& x, const SparseMatrix& A, const Vector& y, SparseMatrix& B) const { static const sparse::LinearAlgebraGeneric generic; generic.dsptd(x, A, y, B); } } // namespace sparse } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraTorch.h0000664000175000017500000000217615161702250023302 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraSparse.h" #include "eckit/linalg/detail/Torch.h" namespace eckit::linalg::sparse { struct LinearAlgebraTorch final : public LinearAlgebraSparse, detail::Torch { LinearAlgebraTorch(const std::string& name, torch::DeviceType device, torch::ScalarType scalar = torch::kFloat64) : LinearAlgebraSparse(name), Torch(device, scalar) {} void spmv(const SparseMatrix&, const Vector&, Vector&) const override; void spmm(const SparseMatrix&, const Matrix&, Matrix&) const override; void dsptd(const Vector&, const SparseMatrix&, const Vector&, SparseMatrix&) const override; void print(std::ostream& os) const override { Torch::print(os); } }; } // namespace eckit::linalg::sparse eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraMKL.cc0000664000175000017500000000730515161702250023003 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/sparse/LinearAlgebraMKL.h" #include "mkl.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/SparseMatrix.h" #include "eckit/linalg/Vector.h" #include "eckit/linalg/sparse/LinearAlgebraGeneric.h" namespace eckit { namespace linalg { namespace sparse { static const LinearAlgebraMKL __la("mkl"); static const double alpha = 1.; static const double beta = 0.; void LinearAlgebraMKL::print(std::ostream& out) const { out << "LinearAlgebraMKL[]"; } void LinearAlgebraMKL::spmv(const SparseMatrix& A, const Vector& x, Vector& y) const { ASSERT(x.size() == A.cols()); ASSERT(y.size() == A.rows()); // We expect indices to be 0-based ASSERT(A.outer()[0] == 0); const auto m = static_cast(A.rows()); const auto k = static_cast(A.cols()); // FIXME: mkl_dcsrmv is deprecated, use mkl_sparse_d_mv instead // void mkl_dcsrmv(const char *transa, const MKL_INT *m, const MKL_INT *k, // const double *alpha, const char *matdescra, // const double *val, const MKL_INT *indx, const MKL_INT *pntrb, const MKL_INT *pntre, // const double *x, const double *beta, double *y); const auto* matrix = static_cast(A.data()); const auto* inner = static_cast(A.inner()); const auto* outer = static_cast(A.outer()); const auto* vector = static_cast(x.data()); mkl_dcsrmv("N", &m, &k, &alpha, "G__C", matrix, inner, outer, outer + 1, vector, &beta, y.data()); } void LinearAlgebraMKL::spmm(const SparseMatrix& A, const Matrix& B, Matrix& C) const { ASSERT(A.cols() == B.rows()); ASSERT(A.rows() == C.rows()); ASSERT(B.cols() == C.cols()); // We expect indices to be 0-based ASSERT(A.outer()[0] == 0); const auto m = static_cast(A.rows()); const auto n = static_cast(C.cols()); const auto k = static_cast(A.cols()); // FIXME: with 0-based indexing, MKL assumes row-major ordering for B and C // We need to use 1-based indexing i.e. offset outer and inner indices by 1 std::vector pntrb(A.rows() + 1); for (Size i = 0; i < A.rows() + 1; ++i) { pntrb[i] = A.outer()[i] + 1; } std::vector indx(A.nonZeros()); for (Size i = 0; i < A.nonZeros(); ++i) { indx[i] = A.inner()[i] + 1; } // FIXME: mkl_dcsrmm is deprecated, use mkl_sparse_d_mm instead // void mkl_dcsrmm(const char *transa, const MKL_INT *m, const MKL_INT *n, const MKL_INT *k, // const double *alpha, const char *matdescra, // const double *val, const MKL_INT *indx, const MKL_INT *pntrb, const MKL_INT *pntre, // const double *b, const MKL_INT *ldb, const double *beta, // double *c, const MKL_INT *ldc); mkl_dcsrmm("N", &m, &n, &k, &alpha, "G__F", A.data(), indx.data(), pntrb.data(), pntrb.data() + 1, B.data(), &k, &beta, C.data(), &m); } void LinearAlgebraMKL::dsptd(const Vector& x, const SparseMatrix& A, const Vector& y, SparseMatrix& B) const { static const sparse::LinearAlgebraGeneric generic; generic.dsptd(x, A, y, B); } } // namespace sparse } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/sparse/LinearAlgebraGeneric.cc0000664000175000017500000000701315161702250023730 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/sparse/LinearAlgebraGeneric.h" #include #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/SparseMatrix.h" #include "eckit/linalg/Vector.h" namespace eckit::linalg::sparse { static const LinearAlgebraGeneric __la_generic("generic"); #if eckit_HAVE_OMP static const LinearAlgebraGeneric __la_openmp("openmp"); #endif void LinearAlgebraGeneric::print(std::ostream& out) const { out << "LinearAlgebraGeneric[]"; } void LinearAlgebraGeneric::spmv(const SparseMatrix& A, const Vector& x, Vector& y) const { const auto Ni = A.rows(); const auto Nj = A.cols(); ASSERT(y.rows() == Ni); ASSERT(x.rows() == Nj); if (A.empty()) { return; } const auto* const outer = A.outerIndex(); const auto* const inner = A.inner(); const auto* const val = A.data(); ASSERT(outer[0] == 0); // expect indices to be 0-based #if eckit_HAVE_OMP #pragma omp parallel for #endif for (Size i = 0; i < Ni; ++i) { Scalar sum = 0.; for (auto c = outer[i]; c < outer[i + 1]; ++c) { sum += val[c] * x[static_cast(inner[c])]; } y[i] = sum; } } void LinearAlgebraGeneric::spmm(const SparseMatrix& A, const Matrix& B, Matrix& C) const { const auto Ni = A.rows(); const auto Nj = A.cols(); const auto Nk = B.cols(); ASSERT(C.rows() == Ni); ASSERT(B.rows() == Nj); ASSERT(C.cols() == Nk); if (A.empty()) { return; } const auto* const outer = A.outerIndex(); const auto* const inner = A.inner(); const auto* const val = A.data(); ASSERT(outer[0] == 0); // expect indices to be 0-based std::vector sum; #if eckit_HAVE_OMP #pragma omp parallel #endif { std::vector sum(Nk); #if eckit_HAVE_OMP #pragma omp for #endif for (Size i = 0; i < Ni; ++i) { sum.assign(Nk, 0); for (auto c = outer[i]; c < outer[i + 1]; ++c) { const auto j = static_cast(inner[c]); const auto v = val[c]; for (Size k = 0; k < Nk; ++k) { sum[k] += v * B(j, k); } } for (Size k = 0; k < Nk; ++k) { C(i, k) = sum[k]; } } } } void LinearAlgebraGeneric::dsptd(const Vector& x, const SparseMatrix& A, const Vector& y, SparseMatrix& B) const { const auto Ni = A.rows(); const auto Nj = A.cols(); ASSERT(x.size() == Ni); ASSERT(y.size() == Nj); B = A; if (A.empty()) { return; } const auto* const outer = B.outer(); const auto* const inner = B.inner(); auto* const val = const_cast(B.data()); ASSERT(outer[0] == 0); // expect indices to be 0-based #if eckit_HAVE_OMP #pragma omp parallel for #endif for (Size i = 0; i < Ni; ++i) { for (auto k = outer[i]; k < outer[i + 1]; ++k) { const auto j = static_cast(inner[k]); ASSERT(j < Nj); val[k] *= x[i] * y[j]; } } } } // namespace eckit::linalg::sparse eckit-2.0.7/src/eckit/linalg/LinearAlgebraDense.h0000664000175000017500000000403715161702250021762 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/linalg/types.h" namespace eckit::linalg { //----------------------------------------------------------------------------- class LinearAlgebraDense { public: // - Static methods /// Get backend, re-setting default static const LinearAlgebraDense& backend(const std::string& name = ""); /// Get backend static const LinearAlgebraDense& getBackend(const std::string& name); /// Check if a backend is available static bool hasBackend(const std::string& name); /// List all available backends static std::ostream& list(std::ostream&); /// Return active backend name static const std::string& name(); // - Methods /// Compute the inner product of vectors x and y virtual Scalar dot(const Vector& x, const Vector& y) const = 0; /// Compute the product of a dense matrix A and vector x /// @note y must be allocated and sized correctly virtual void gemv(const Matrix& A, const Vector& x, Vector& y) const = 0; /// Compute the product of dense matrices A and X /// @note Y must be allocated and sized correctly virtual void gemm(const Matrix& A, const Matrix& X, Matrix& Y) const = 0; protected: LinearAlgebraDense() = default; LinearAlgebraDense(const std::string& name); virtual ~LinearAlgebraDense() = default; private: virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream& s, const LinearAlgebraDense& p) { p.print(s); return s; } }; //----------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/Triplet.cc0000664000175000017500000000200515161702250020065 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/Triplet.h" #include namespace eckit::linalg { //---------------------------------------------------------------------------------------------------------------------- void Triplet::print(std::ostream& os) const { os << "Triplet[" << "row=" << row_ << "," << "col=" << col_ << "," << "val=" << val_ << "]"; } std::ostream& operator<<(std::ostream& os, const eckit::linalg::Triplet& p) { p.print(os); return os; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/LinearAlgebraSparse.h0000664000175000017500000000434215161702250022160 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/linalg/types.h" namespace eckit::linalg { //----------------------------------------------------------------------------- class LinearAlgebraSparse { public: // - Static methods /// Get backend, re-setting default static const LinearAlgebraSparse& backend(const std::string& name = ""); /// Get backend static const LinearAlgebraSparse& getBackend(const std::string& name); /// Check if a backend is available static bool hasBackend(const std::string& name); /// List all available backends static std::ostream& list(std::ostream&); /// Return active backend name static const std::string& name(); // - Methods /// Compute the product of a sparse matrix A and vector x /// @note y must be allocated and sized correctly virtual void spmv(const SparseMatrix& A, const Vector& x, Vector& y) const = 0; /// Compute the product of sparse matrix A and dense matrix X /// @note Y must be allocated and sized correctly virtual void spmm(const SparseMatrix& A, const Matrix& X, Matrix& Y) const = 0; /// Compute the product x A' y with x and y diagonal matrices stored as /// vectors and A a sparse matrix /// @note B does NOT need to be allocated/sized correctly virtual void dsptd(const Vector& x, const SparseMatrix& A, const Vector& y, SparseMatrix& B) const = 0; protected: LinearAlgebraSparse() = default; LinearAlgebraSparse(const std::string& name); virtual ~LinearAlgebraSparse() = default; private: virtual void print(std::ostream&) const = 0; friend std::ostream& operator<<(std::ostream& s, const LinearAlgebraSparse& p) { p.print(s); return s; } }; //----------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/SparseMatrix.cc0000664000175000017500000003706715161702250021104 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/SparseMatrix.h" #include #include #include #include #include #include "eckit/eckit.h" // for endianness #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/AutoCloser.h" #include "eckit/io/MemoryHandle.h" #include "eckit/linalg/allocator/BufferAllocator.h" #include "eckit/linalg/allocator/StandardAllocator.h" #include "eckit/log/Log.h" #include "eckit/memory/MemoryBuffer.h" #include "eckit/serialisation/FileStream.h" #include "eckit/serialisation/Stream.h" namespace eckit::linalg { // Extend range of Index for unsigned compression indices (should only be positive), preserving binary compatibility static_assert(sizeof(Index) == sizeof(SparseMatrix::UIndex), "sizeof(sizeof(Index) == SparseMatrix::UIndex)"); static constexpr bool littleEndian = eckit_LITTLE_ENDIAN != 0; //---------------------------------------------------------------------------------------------------------------------- void SparseMatrix::Shape::print(std::ostream& os) const { os << "Shape[" << "nnz=" << size_ << "," << "rows=" << rows_ << "," << "cols=" << cols_ << "]"; } SparseMatrix::SparseMatrix(Allocator* alloc) : owner_(alloc != nullptr ? alloc : new allocator::StandardAllocator) { spm_ = owner_->allocate(shape_); } SparseMatrix::SparseMatrix(Size rows, Size cols, Allocator* alloc) : owner_(alloc != nullptr ? alloc : new allocator::StandardAllocator) { reserve(rows, cols, 1); } SparseMatrix::SparseMatrix(Size rows, Size cols, const std::vector& triplets) : owner_(new allocator::StandardAllocator) { // Count number of non-zeros, allocate memory 1 triplet per non-zero Size nnz = std::count_if(triplets.begin(), triplets.end(), [](const auto& tri) { return tri.nonZero(); }); if (auto max = static_cast(std::numeric_limits::max()); max < nnz) { throw OutOfRange("SparseMatrix::SparseMatrix: too many non-zero entries, nnz=" + std::to_string(nnz) + ", max=" + std::to_string(max), Here()); } reserve(rows, cols, nnz); Size pos = 0; Size row = 0; spm_.outer_[0] = 0; /* first entry (base) is always zero */ // Build vectors of inner indices and values, update outer index per row for (const auto& tri : triplets) { if (tri.nonZero()) { // triplets are ordered by rows ASSERT(tri.row() >= row); ASSERT(tri.row() < shape_.rows_); ASSERT(tri.col() < shape_.cols_); // start a new row while (tri.row() > row) { spm_.outer_[++row] = static_cast(pos); } spm_.inner_[pos] = static_cast(tri.col()); spm_.data_[pos] = tri.value(); ++pos; } } while (row < shape_.rows_) { spm_.outer_[++row] = static_cast(pos); } ASSERT(static_cast(spm_.outer_[shape_.outerSize() - 1]) == nonZeros()); } SparseMatrix::SparseMatrix(Stream& s) : owner_(new allocator::StandardAllocator) { decode(s); } SparseMatrix::SparseMatrix(const MemoryBuffer& buffer) : owner_(new allocator::BufferAllocator(buffer)) { spm_ = owner_->allocate(shape_); } SparseMatrix::SparseMatrix(const SparseMatrix& other) : owner_(new allocator::StandardAllocator) { if (!other.empty()) { // in case we copy an other that was constructed empty reserve(other.rows(), other.cols(), other.nonZeros()); std::memcpy(spm_.data_, other.spm_.data_, shape_.sizeofData()); std::memcpy(spm_.outer_, other.spm_.outer_, shape_.sizeofOuter()); std::memcpy(spm_.inner_, other.spm_.inner_, shape_.sizeofInner()); } } SparseMatrix::SparseMatrix(SparseMatrix&& other) { swap(other); } SparseMatrix& SparseMatrix::operator=(const SparseMatrix& other) { SparseMatrix copy(other); swap(copy); return *this; } SparseMatrix& SparseMatrix::operator=(SparseMatrix&& other) { swap(other); return *this; } SparseMatrix::~SparseMatrix() { reset(); } void SparseMatrix::reset() { if (owner_) { owner_->deallocate(spm_, shape_); } spm_.reset(); shape_.reset(); } void SparseMatrix::reserve(Size rows, Size cols, Size nnz) { ASSERT(nnz > 0); ASSERT(nnz <= rows * cols); ASSERT(rows > 0 && cols > 0); reset(); shape_.rows_ = rows; shape_.cols_ = cols; shape_.size_ = nnz; spm_ = owner_->allocate(shape_); } void SparseMatrix::save(const PathName& path) const { FileStream s(path, "w"); auto c = closer(s); encode(s); } void SparseMatrix::load(const PathName& path) { FileStream s(path, "r"); auto c = closer(s); decode(s); } struct SPMInfo { size_t size_; ///< non-zeros size_t rows_; ///< rows size_t cols_; ///< columns ptrdiff_t data_; ptrdiff_t outer_; ptrdiff_t inner_; }; void SparseMatrix::load(const void* buffer, size_t bufferSize, Layout& layout, Shape& shape) { const auto* b = static_cast(buffer); MemoryHandle mh(buffer, bufferSize); mh.openForRead(); struct SPMInfo info; mh.read(&info, sizeof(SPMInfo)); ASSERT(info.size_ && info.rows_ && info.cols_); ASSERT(info.data_ > 0 && info.outer_ > 0 && info.inner_ > 0); // check that shape is matching what we are loading shape.size_ = info.size_; shape.rows_ = info.rows_; shape.cols_ = info.cols_; Log::debug() << "Loading matrix from buffer: " << " rows " << shape.rows_ << " cols " << shape.cols_ << " nnzs " << shape.size_ << " allocSize " << shape.allocSize() << std::endl; ASSERT(bufferSize >= sizeof(SPMInfo) + shape.sizeofData() + shape.sizeofOuter() + shape.sizeofInner()); auto* addr = const_cast(b); layout.data_ = reinterpret_cast(addr + info.data_); layout.outer_ = reinterpret_cast(addr + info.outer_); layout.inner_ = reinterpret_cast(addr + info.inner_); // check offsets don't segfault ASSERT(info.data_ + shape.sizeofData() <= bufferSize); ASSERT(info.outer_ + shape.sizeofOuter() <= bufferSize); ASSERT(info.inner_ + shape.sizeofInner() <= bufferSize); } void SparseMatrix::dump(MemoryBuffer& buffer) const { SparseMatrix::dump(buffer.data(), buffer.size()); } void SparseMatrix::dump(void* buffer, size_t size) const { size_t minimum = sizeof(SPMInfo) + shape_.sizeofData() + shape_.sizeofOuter() + shape_.sizeofInner(); ASSERT(size >= minimum); MemoryHandle mh(buffer, size); mh.openForWrite(size); SPMInfo info; info.size_ = nonZeros(); info.rows_ = rows(); info.cols_ = cols(); info.data_ = sizeof(SPMInfo); info.outer_ = info.data_ + shape_.sizeofData(); info.inner_ = info.outer_ + shape_.sizeofOuter(); Log::debug() << "Dumping matrix : " << " rows " << info.rows_ << " cols " << info.cols_ << " nnzs " << info.size_ << " allocSize " << shape_.allocSize() << std::endl; /// @todo we should try to get these memory aligned (to say 64 bytes) mh.write(&info, sizeof(SPMInfo)); ASSERT(mh.write(spm_.data_, shape_.sizeofData()) == static_cast(shape_.sizeofData())); ASSERT(mh.write(spm_.outer_, shape_.sizeofOuter()) == static_cast(shape_.sizeofOuter())); ASSERT(mh.write(spm_.inner_, shape_.sizeofInner()) == static_cast(shape_.sizeofInner())); } void SparseMatrix::swap(SparseMatrix& other) { std::swap(spm_, other.spm_); std::swap(shape_, other.shape_); owner_.swap(other.owner_); } const Index* SparseMatrix::outer() const { if (auto max = static_cast(std::numeric_limits::max()); max < nonZeros()) { throw OutOfRange("SparseMatrix::outer: too many non-zero entries, nnz=" + std::to_string(nonZeros()) + ", max=" + std::to_string(max) + " (for Index-typed compatibility)", Here()); } return reinterpret_cast(spm_.outer_); } void SparseMatrix::cols(Size cols) { ASSERT(cols > 0); shape_.cols_ = cols; } size_t SparseMatrix::footprint() const { return sizeof(*this) + shape_.allocSize(); } bool SparseMatrix::inSharedMemory() const { ASSERT(owner_); return owner_->inSharedMemory(); } void SparseMatrix::dump(std::ostream& os) const { for (Size i = 0; i < rows(); ++i) { const_iterator itr = begin(i); const_iterator iend = end(i); if (itr == iend) { continue; } os << itr.row(); for (; itr != iend; ++itr) { os << " " << itr.col() << " " << *itr; } os << std::endl; } } void SparseMatrix::print(std::ostream& os) const { os << "SparseMatrix[" << shape_ << "," << *owner_ << "]"; } SparseMatrix& SparseMatrix::setIdentity(Size rows, Size cols) { ASSERT(rows > 0 && cols > 0); auto nnz = std::min(rows, cols); reserve(rows, cols, nnz); for (Size i = 0; i < nnz; ++i) { spm_.outer_[i] = static_cast(i); spm_.inner_[i] = static_cast(i); } for (Size i = nnz; i <= shape_.rows_; ++i) { spm_.outer_[i] = static_cast(nnz); } for (Size i = 0; i < shape_.size_; ++i) { spm_.data_[i] = static_cast(1); } return *this; } SparseMatrix& SparseMatrix::transpose() { /// @note Can SparseMatrix::transpose() be done more efficiently? /// We are building another matrix and then swapping std::vector triplets; triplets.reserve(nonZeros()); for (Size r = 0; r < shape_.rows_; ++r) { for (auto c = spm_.outer_[r]; c < spm_.outer_[r + 1]; ++c) { ASSERT(spm_.inner_[c] >= 0); triplets.emplace_back(static_cast(spm_.inner_[c]), r, spm_.data_[c]); } } std::sort(triplets.begin(), triplets.end()); // triplets must be sorted by row SparseMatrix tmp(shape_.cols_, shape_.rows_, triplets); swap(tmp); return *this; } SparseMatrix SparseMatrix::rowReduction(const std::vector& p) const { ASSERT(p.size() <= rows()); std::vector triplets; for (size_t newrow = 0; newrow < p.size(); ++newrow) { size_t row = p[newrow]; for (auto itr = begin(row), iend = end(row); itr != iend; ++itr) { triplets.emplace_back(newrow, itr.col(), *itr); } } return {p.size(), cols(), triplets}; } SparseMatrix& SparseMatrix::prune(Scalar val) { std::vector v; std::vector inner; Size nnz = 0; for (Size r = 0; r < shape_.rows_; ++r) { const auto start = spm_.outer_[r]; spm_.outer_[r] = static_cast(nnz); for (auto c = start; c < spm_.outer_[r + 1]; ++c) { if (spm_.data_[c] != val) { v.push_back(spm_.data_[c]); inner.push_back(spm_.inner_[c]); ++nnz; } } } spm_.outer_[shape_.rows_] = static_cast(nnz); SparseMatrix tmp; tmp.reserve(shape_.rows_, shape_.cols_, nnz); std::memcpy(tmp.spm_.data_, v.data(), nnz * sizeof(Scalar)); std::memcpy(tmp.spm_.outer_, spm_.outer_, shape_.outerSize() * sizeof(UIndex)); std::memcpy(tmp.spm_.inner_, inner.data(), nnz * sizeof(Index)); swap(tmp); return *this; } const SparseMatrix::Allocator& SparseMatrix::owner() const { ASSERT(owner_); return *owner_; } void SparseMatrix::encode(Stream& s) const { s << shape_.rows_; s << shape_.cols_; s << shape_.size_; s << littleEndian; s << sizeof(Index); s << sizeof(Scalar); s << sizeof(Size); Log::debug() << "Encoding matrix : " << " rows " << rows() << " cols " << cols() << " nnz " << nonZeros() << " footprint " << footprint() << std::endl; s.writeLargeBlob(spm_.outer_, shape_.outerSize() * sizeof(UIndex)); s.writeLargeBlob(spm_.inner_, shape_.innerSize() * sizeof(Index)); s.writeLargeBlob(spm_.data_, shape_.dataSize() * sizeof(Scalar)); } void SparseMatrix::decode(Stream& s) { Size rows = 0; Size cols = 0; Size nnz = 0; s >> rows; s >> cols; s >> nnz; bool little_endian = true; s >> little_endian; ASSERT(littleEndian == little_endian); size_t index_size = 0; s >> index_size; ASSERT(index_size == sizeof(Index)); size_t scalar_size = 0; s >> scalar_size; ASSERT(scalar_size == sizeof(Scalar)); size_t size_size = 0; s >> size_size; ASSERT(size_size == sizeof(Size)); reset(); owner_ = std::make_unique(); reserve(rows, cols, nnz); Log::debug() << "Decoding matrix : " << " rows " << rows << " cols " << cols << " nnz " << nnz << " footprint " << footprint() << std::endl; s.readLargeBlob(spm_.outer_, shape_.outerSize() * sizeof(UIndex)); s.readLargeBlob(spm_.inner_, shape_.innerSize() * sizeof(Index)); s.readLargeBlob(spm_.data_, shape_.dataSize() * sizeof(Scalar)); } Stream& operator<<(Stream& s, const SparseMatrix& v) { v.encode(s); return s; } SparseMatrix::const_iterator SparseMatrix::const_iterator::operator++(int) { auto it = *this; ++(*this); return it; } bool SparseMatrix::const_iterator::operator==(const SparseMatrix::const_iterator& other) const { ASSERT(other.matrix_ == matrix_); return other.index_ == index_; } SparseMatrix::const_iterator::const_iterator(const SparseMatrix& matrix) : matrix_(const_cast(&matrix)), index_(0) { for (row_ = 0; matrix_->spm_.outer_[row_ + 1] == 0;) { ++row_; } } SparseMatrix::const_iterator::const_iterator(const SparseMatrix& matrix, Size row) : matrix_(const_cast(&matrix)), row_(row) { if (const Size rows = matrix_->rows(); row_ > rows) { row_ = rows; } index_ = static_cast(matrix_->spm_.outer_[row_]); } Size SparseMatrix::const_iterator::col() const { ASSERT(matrix_ && index_ < matrix_->nonZeros()); return static_cast(matrix_->inner()[index_]); } Size SparseMatrix::const_iterator::row() const { return row_; } SparseMatrix::const_iterator& SparseMatrix::const_iterator::operator++() { if (lastOfRow()) { row_++; } index_++; return *this; } const Scalar& SparseMatrix::const_iterator::operator*() const { assert(matrix_ && index_ < matrix_->nonZeros()); return matrix_->data()[index_]; } void SparseMatrix::const_iterator::print(std::ostream& os) const { os << "SparseMatrix::iterator(row=" << row_ << ", col=" << col() << ", index=" << index_ << ", value=" << operator*() << ")" << std::endl; } Scalar& SparseMatrix::iterator::operator*() { assert(matrix_ && index_ < matrix_->nonZeros()); return matrix_->spm_.data_[index_]; } //---------------------------------------------------------------------------------------------------------------------- SparseMatrix::Allocator::~Allocator() = default; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/LinearAlgebra.h0000664000175000017500000000731015161702250021000 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/linalg/LinearAlgebraDense.h" #include "eckit/linalg/LinearAlgebraSparse.h" #include "eckit/linalg/types.h" namespace eckit::linalg { //----------------------------------------------------------------------------- class LinearAlgebra { public: // - Static methods /// List all available backends static std::ostream& list(std::ostream&); /// Check if a backend is available static bool hasBackend(const std::string& name); /// Get current or specific LinearAlgebraDense backend static const LinearAlgebraDense& getDenseBackend(const std::string& name = ""); /// Get current or specific LinearAlgebraSparse backend static const LinearAlgebraSparse& getSparseBackend(const std::string& name = ""); /// Get current or specific LinearAlgebraDense backend (re-setting default) static const LinearAlgebraDense& denseBackend(const std::string& name = ""); /// Get current or specific LinearAlgebraSparse backend (re-setting default) static const LinearAlgebraSparse& sparseBackend(const std::string& name = ""); /// Check if a LinearAlgebraDense backend is available static bool hasDenseBackend(const std::string& name); /// Check if a LinearAlgebraSparse backend is available static bool hasSparseBackend(const std::string& name); /// Compute the inner product of vectors x and y static Scalar dot(const Vector& x, const Vector& y) { return LinearAlgebraDense::backend().dot(x, y); } /// Compute the product of a dense matrix A and vector x /// @note y must be allocated and sized correctly static void gemv(const Matrix& A, const Vector& x, Vector& y) { LinearAlgebraDense::backend().gemv(A, x, y); } /// Compute the product of dense matrices A and X /// @note Y must be allocated and sized correctly static void gemm(const Matrix& A, const Matrix& X, Matrix& Y) { LinearAlgebraDense::backend().gemm(A, X, Y); } /// Compute the product of a sparse matrix A and vector x /// @note y must be allocated and sized correctly static void spmv(const SparseMatrix& A, const Vector& x, Vector& y) { LinearAlgebraSparse::backend().spmv(A, x, y); } /// Compute the product of sparse matrix A and dense matrix X /// @note Y must be allocated and sized correctly static void spmm(const SparseMatrix& A, const Matrix& X, Matrix& Y) { LinearAlgebraSparse::backend().spmm(A, X, Y); } /// Compute the product x A' y with x and y diagonal matrices stored as /// vectors and A a sparse matrix /// @note B does NOT need to be allocated/sized correctly static void dsptd(const Vector& x, const SparseMatrix& A, const Vector& y, SparseMatrix& B) { LinearAlgebraSparse::backend().dsptd(x, A, y, B); } protected: LinearAlgebra() = default; private: LinearAlgebra(const LinearAlgebra&) = delete; LinearAlgebra& operator=(const LinearAlgebra&) = delete; friend std::ostream& operator<<(std::ostream& s, const LinearAlgebra&) { return s << "LinearAlgebra[LinearAlgebraDense=[" << LinearAlgebraDense::backend() << "],LinearAlgebraSparse=[" << LinearAlgebraSparse::backend() << "]]"; } }; //----------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/Vector.h0000664000175000017500000000570115161702250017554 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ /// @author Florian Rathgeber /// @author Pedro Maciel /// @author Tiago Quintino #pragma once #include "eckit/linalg/types.h" namespace eckit { class Stream; } namespace eckit::linalg { //---------------------------------------------------------------------------------------------------------------------- /// Vector for Linear Algebra operations /// @todo provide a const view class Vector { public: // methods // -- Constructors /// Default constructor (empty vector) Vector(); /// Construct vector of given size (allocates memory, not initialised) Vector(Size length); /// Construct vector from existing data (does NOT take ownership) Vector(const Scalar array[], Size length); /// Constructor from Stream Vector(Stream&); /// Copy constructor Vector(const Vector&); ~Vector(); // -- Mutators Vector& operator=(const Vector&); /// Swap this vector for another void swap(Vector&); /// Resize vector to given size (invalidates data) void resize(Size length); /// Set data to zero void setZero(); /// Fill vector with given scalar void fill(Scalar); // -- Serialisation /// Serialise to a Stream void encode(Stream&) const; // -- Accessors /// @returns size (rows * cols) Size size() const { return length_; } /// @returns number of rows (i.e. size) Size rows() const { return length_; } /// @returns number of columns (always 1) Size cols() const { return 1; } Scalar& operator[](Size i) { return array_[i]; } const Scalar& operator[](Size i) const { return array_[i]; } /// @returns modifiable view of the data Scalar* data() { return array_; } /// @returns read-only view of the data const Scalar* data() const { return array_; } /// @returns iterator to beginning of the data Scalar* begin() { return array_; } /// @returns const iterator to beginning of the data const Scalar* begin() const { return array_; } /// @returns iterator to end of the data Scalar* end() { return array_ + length_; } /// @returns const iterator to end of the data const Scalar* end() const { return array_ + length_; } protected: // member variables Scalar* array_; ///< Container Size length_; ///< Vector length/size bool own_; ///< do we own the memory allocated in the container ? }; Stream& operator<<(Stream&, const Vector&); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/Matrix.h0000664000175000017500000000625515161702250017563 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/types.h" namespace eckit { class Stream; } namespace eckit::linalg { //----------------------------------------------------------------------------- /// Dense matrix in column major storage order class Matrix { public: // types using Size = linalg::Size; public: // methods // -- Constructors /// Default constructor (empty matrix) Matrix(); /// Construct matrix with given rows and columns (allocates memory, not initialised) Matrix(Size rows, Size cols); /// Construct matrix from existing data (does NOT take ownership) Matrix(const Scalar* array, Size rows, Size cols); /// Constructor from Stream Matrix(Stream&); /// Copy constructor Matrix(const Matrix&); /// Destructor ~Matrix(); // -- Mutators Matrix& operator=(const Matrix&); /// Swap this matrix with another void swap(Matrix&); /// Resize matrix to given number of rows/columns (invalidates data) void resize(Size rows, Size cols); /// Set data to zero void setZero(); /// Fill vector with given scalar void fill(Scalar); // -- Serialisation /// Serialise to a Stream void encode(Stream&) const; // -- Accessors /// @returns size (rows * cols) Size size() const { return rows_ * cols_; } /// @returns number of rows Size rows() const { return rows_; } /// @returns number of columns Size cols() const { return cols_; } /// Access by row and column /// @note implements column-major (Fortran-style) ordering Scalar& operator()(Size row, Size col) { return array_[col * rows_ + row]; } const Scalar& operator()(Size row, Size col) const { return array_[col * rows_ + row]; } /// Access to linearised storage Scalar& operator[](Size i) { return array_[i]; } const Scalar& operator[](Size i) const { return array_[i]; } /// @returns modifiable view of the data Scalar* data() { return array_; } /// @returns read-only view of the data const Scalar* data() const { return array_; } /// @returns iterator to beginning of the data Scalar* begin() { return array_; } /// @returns const iterator to beginning of the data const Scalar* begin() const { return array_; } /// @returns iterator to end of the data Scalar* end() { return array_ + size(); } /// @returns const iterator to end of the data const Scalar* end() const { return array_ + size(); } protected: // member variables /// Container Scalar* array_; /// Number of rows Size rows_; /// Number of columns Size cols_; /// Indicate ownership bool own_; }; Stream& operator<<(Stream&, const Matrix&); //----------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/Tensor.h0000664000175000017500000003330715161702250017567 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ /// @file Tensor.h /// @author Tiago Quintino /// @date May 2021 #pragma once #include #include #include #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/linalg/types.h" #include "eckit/serialisation/Stream.h" #include "eckit/types/Types.h" namespace eckit::linalg { //---------------------------------------------------------------------------------------------------------------------- /// Dense Tensor in column major storage order /// This class is not meant to be accessed 1 element at a time, as its access isn't efficient enough in tight loops. /// It is meant to be passed as a contiguous piece of data into linear algebra packages. /// The operator() is only provided for inspection and testing. /// /// Supports 2 memory layouts: /// * Column-major (as in Fortran) where [fast idx] .... [slow idx] - also known as "left" layout /// * Row-major (as in C) where [slow idx] .... [fast idx] - also known as "right" layout template class Tensor { public: // class methods enum class Layout : int { // specify underlying type to be "int" for interoperability Right = 0, Left = 1, RowMajor = Right, ColMajor = Left }; static Size flatSize(const std::vector& shape) { return std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies()); } static std::vector strides(Layout layout, const std::vector& shape) { std::vector s(shape.size()); if (layout == Layout::ColMajor) { Size prod = 1; s[0] = prod; for (int i = 1; i < s.size(); ++i) { prod *= shape[i - 1]; s[i] = prod; } } else { Size tmp = 1; for (int i = shape.size() - 1; i >= 1; i--) { tmp *= shape[i]; s[i - 1] = tmp; } s[shape.size() - 1] = 1; } return s; } public: // methods /// Default constructor (empty tensor) Tensor(Layout layout = Layout::ColMajor) : array_{nullptr}, size_(0), shape_(0), strides_(0), layout_(layout), own_(false) {} /// Construct tensor with given rows and columns (allocates memory, not initialised) Tensor(const std::vector& shape, Layout layout = Layout::ColMajor) : array_{nullptr}, shape_(shape), strides_(strides(layout, shape)), layout_(layout), own_(true) { size_ = flatSize(shape_); ASSERT(size() > 0); array_ = new S[size_]; ASSERT(array_); } /// Construct tensor from existing data (does NOT take ownership) Tensor(S* array, const std::vector& shape, Layout layout = Layout::ColMajor) : array_(array), strides_(strides(layout, shape)), layout_(layout), own_(false) { shape_ = shape; size_ = flatSize(shape_); ASSERT(size() > 0); ASSERT(array_); } /// Constructor from Stream Tensor(Stream& s) : array_{nullptr}, size_(0), shape_(0), own_(true) { Size shape_size; // layout int layoutAsInt; s >> layoutAsInt; layout_ = static_cast(layoutAsInt); // shape s >> shape_size; shape_.resize(shape_size); for (auto& v : shape_) { s >> v; } resize(shape_); ASSERT(size() > 0); ASSERT(array_); // data s.readBlob(array_, size() * sizeof(S)); strides_ = strides(layout_, shape_); } /// Copy constructor Tensor(const Tensor& other) : array_(new S[other.size()]), size_(other.size_), shape_(other.shape_), strides_(other.strides_), layout_(other.layout_), own_(true) { ASSERT(size() > 0); ASSERT(array_); ::memcpy(array_, other.array_, size() * sizeof(S)); } /// Move constructor Tensor(Tensor&& other) noexcept { shape_ = std::move(other.shape_); strides_ = std::move(other.strides_); size_ = other.size_; layout_ = other.layout_; own_ = other.own_; array_ = other.array_; // nullify moved-from tensor other.array_ = nullptr; other.own_ = false; other.shape_.clear(); other.strides_.clear(); other.size_ = 0; } /// Destructor ~Tensor() { if (own_ && array_) { delete[] array_; } } /// Assignment /// Consistently retains ownership to avoid surprises in ownership behaviour. /// However, if a resize() triggers reallocation of memory, ownership will be reset. Tensor& operator=(const Tensor& other) { layout_ = other.layout_; resize(other.shape()); // shape & strides already correct // ownership remains same ::memcpy(array_, other.array_, size() * sizeof(S)); return *this; } /// Move assignment operator Tensor& operator=(Tensor&& other) noexcept { if (&other != this) { if (own_ && array_) { delete[] array_; } shape_ = std::move(other.shape_); strides_ = std::move(other.strides_); size_ = other.size_; layout_ = other.layout_; own_ = other.own_; array_ = other.array_; // nullify moved-from tensor other.array_ = nullptr; other.own_ = false; other.shape_.clear(); other.strides_.clear(); other.size_ = 0; } return *this; } /// Swap this tensor with another void swap(Tensor& other) { std::swap(array_, other.array_); std::swap(size_, other.size_); std::swap(shape_, other.shape_); std::swap(strides_, other.strides_); std::swap(layout_, other.layout_); std::swap(own_, other.own_); } /// Resize tensor to given a shape /// Invalidates data if shapes don't match, otherwise keeps data and simply reshapes void resize(const std::vector& shape) { if (this->size() != flatSize(shape)) { // avoid reallocation if same size Tensor m(shape, layout_); swap(m); } else { // optimise when we dont need to reallocate shape_ = shape; strides_ = strides(layout_, shape); } } /// Set data to zero void zero() { ASSERT(size() > 0); ASSERT(array_); ::memset(array_, 0, size() * sizeof(S)); } /// Fill vector with given S void fill(S value) { for (Size i = 0; i < size(); ++i) { array_[i] = value; } } /// Serialise to a Stream /// This serialisation is not cross-platform void encode(Stream& s) const { s << static_cast(layout_); s << shape_.size(); for (auto v : shape_) { s << v; } s.writeBlob(array_, size() * sizeof(S)); } /// @returns flatten size (= product of shape vector) Size size() const { return size_; } /// @returns shape std::vector shape() const { return shape_; } /// Access to linearised storage S& operator[](Size i) { return array_[i]; } const S& operator[](Size i) const { return array_[i]; } /// @returns modifiable view of the data S* data() { return array_; } /// @returns read-only view of the data const S* data() const { return array_; } /// @returns iterator to beginning of the data S* begin() { return array_; } /// @returns const iterator to beginning of the data const S* begin() const { return array_; } /// @returns iterator to end of the data S* end() { return array_ + size(); } /// @returns const iterator to end of the data const S* end() const { return array_ + size(); } void print(std::ostream& s) const { const char sep = ','; s << "Tensor(layout=" << static_cast(layout_) << sep; s << "shape=["; for (int i = 0; i < shape_.size(); ++i) { s << shape_[i] << sep; } s << "],array=["; for (int i = 0; i < size(); ++i) { s << array_[i] << sep; } s << "])"; } Layout layout() const { return layout_; } /// @brief Multidimensional index operator A(i,j,k,...) /// @pre number of parameter must match shape size template S& operator()(Idx... idx) { return array_[index(idx...)]; } /// @brief Multidimensional index operator A(i,j,k,...) /// @pre number of parameter must match shape size template const S& operator()(Idx... idx) const { return array_[index(idx...)]; } /// Transform a colMajor-layout tensor to rowMajor-layout Tensor transformColMajorToRowMajor() const { Tensor r(shape_); // COL-MAJOR to ROW-MAJOR std::vector strd_rev = strides(/*rowMajor*/ Layout::RowMajor, shape_); std::vector strd = strides(/*colMajor*/ Layout::ColMajor, shape_); // main loop Size shape_size = shape_.size(); std::vector col_major_indexes(shape_size); Size gidx_rm; for (int gidx_cm = 0; gidx_cm < size_; gidx_cm++) { // find the tensor indexes from the global index for a CM order for (int idx = 0; idx < shape_size - 1; idx++) { col_major_indexes[idx] = (gidx_cm % strd[idx + 1]) / strd[idx]; } col_major_indexes[shape_size - 1] = gidx_cm / strd[shape_size - 1]; // from the tensor indexes, work out the RM global index gidx_rm = 0; for (int idx = 0; idx < shape_size; idx++) { gidx_rm += col_major_indexes[idx] * strd_rev[idx]; } // assign the corresponding tensor value *(r.data() + gidx_rm) = *(data() + gidx_cm); } // set layout to rowMajor r.layout_ = Layout::RowMajor; // strides are now colMajor-to-rowMajor r.strides_ = strd_rev; return r; } /// Transform a rowMajor-layout tensor to colMajor-layout Tensor transformRowMajorToColMajor() const { Tensor r(shape_); // ROW-MAJOR to COL-MAJOR std::vector strd_rev = strides(/*rowMajor*/ Layout::RowMajor, shape_); std::vector strd = strides(/*colMajor*/ Layout::ColMajor, shape_); Size shape_size = shape_.size(); std::vector row_major_indexes(shape_size); Size gidx_cm; for (int gidx_rm = 0; gidx_rm < size_; gidx_rm++) { // find the tensor indexes from the global index for a RM order row_major_indexes[0] = gidx_rm / strd_rev[0]; for (int idx = 1; idx < shape_size; idx++) { row_major_indexes[idx] = gidx_rm % strd_rev[idx - 1] / strd_rev[idx]; } // from the tensor indexes, work out the CM global index gidx_cm = 0; for (int idx = 0; idx < row_major_indexes.size(); idx++) { gidx_cm += row_major_indexes[idx] * strd[idx]; } // assign the corresponding tensor value *(r.data() + gidx_cm) = *(data() + gidx_rm); } // set the layout as colMajor r.layout_ = Layout::ColMajor; // strides are now rowMajor-to-colMajor r.strides_ = strd; return r; } private: // methods /// compile time variadic template indexing calculation template constexpr Size index_part(Int idx, Ints... next_idx) const { return idx * strides_[Dim] + index_part(next_idx...); } /// compile time variadic template indexing calculation template constexpr Size index_part(Int last_idx) const { return last_idx * strides_[Dim]; } /// compile time variadic template indexing calculation template constexpr Size index(Ints... idx) const { return index_part<0>(idx...); } protected: // member variables S* array_; ///< data Size size_; ///< flattened size std::vector shape_; ///< tensor shape is a vector of sizes per dimension std::vector strides_; ///< tensor strides precomputed at construction Layout layout_; ///< memory layout? (column-major equivalent to Fortran layout) bool own_; ///< ownership }; //---------------------------------------------------------------------------------------------------------------------- using TensorDouble = Tensor; using TensorFloat = Tensor; //---------------------------------------------------------------------------------------------------------------------- template Stream& operator<<(Stream& s, const Tensor& t) { t.encode(s); return s; } template std::ostream& operator<<(std::ostream& s, const Tensor& t) { t.print(s); return s; } inline std::vector shapify(int rank, int shape[]) { std::vector result(rank); for (int i = 0; i < rank; ++i) { result[i] = shape[i]; } return result; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/dense/0000775000175000017500000000000015161702250017234 5ustar alastairalastaireckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraLAPACK.h0000664000175000017500000000166215161702250022756 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraDense.h" namespace eckit::linalg::dense { struct LinearAlgebraLAPACK final : public LinearAlgebraDense { LinearAlgebraLAPACK() {} LinearAlgebraLAPACK(const std::string& name) : LinearAlgebraDense(name) {} Scalar dot(const Vector&, const Vector&) const override; void gemv(const Matrix&, const Vector&, Vector&) const override; void gemm(const Matrix&, const Matrix&, Matrix&) const override; void print(std::ostream&) const override; }; } // namespace eckit::linalg::dense eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraLAPACK.cc0000664000175000017500000000454615161702250023120 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/dense/LinearAlgebraLAPACK.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/Vector.h" extern "C" { double ddot_(const int* n, const double* x, const int* incx, const double* y, const int* incy); void dgemv_(const char* transa, const int* m, const int* n, const double* alpha, const double* a, const int* lda, const double* x, const int* incx, const double* beta, double* y, const int* incy); void dgemm_(const char* transa, const char* transb, const int* m, const int* n, const int* k, const double* alpha, const double* a, const int* lda, const double* b, const int* ldb, const double* beta, double* c, const int* ldc); } namespace eckit::linalg::dense { static const LinearAlgebraLAPACK __la("lapack"); static const char* trans = "N"; static const int inc = 1; static const double alpha = 1.; static const double beta = 0.; void LinearAlgebraLAPACK::print(std::ostream& out) const { out << "LinearAlgebraLAPACK[]"; } Scalar LinearAlgebraLAPACK::dot(const Vector& x, const Vector& y) const { ASSERT(x.size() == y.size()); const auto n = int(x.size()); return ddot_(&n, x.data(), &inc, y.data(), &inc); } void LinearAlgebraLAPACK::gemv(const Matrix& A, const Vector& x, Vector& y) const { ASSERT(x.size() == A.cols()); ASSERT(y.size() == A.rows()); const auto m = int(A.rows()); const auto n = int(A.cols()); dgemv_(trans, &m, &n, &alpha, A.data(), &m, x.data(), &inc, &beta, y.data(), &inc); } void LinearAlgebraLAPACK::gemm(const Matrix& A, const Matrix& B, Matrix& C) const { ASSERT(A.cols() == B.rows()); ASSERT(A.rows() == C.rows()); ASSERT(B.cols() == C.cols()); const auto m = int(A.rows()); const auto n = int(B.cols()); const auto k = int(A.cols()); dgemm_(trans, trans, &m, &n, &k, &alpha, A.data(), &m, B.data(), &k, &beta, C.data(), &m); } } // namespace eckit::linalg::dense eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraGeneric.h0000664000175000017500000000166515161702250023402 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraDense.h" namespace eckit::linalg::dense { struct LinearAlgebraGeneric final : public LinearAlgebraDense { LinearAlgebraGeneric() {} LinearAlgebraGeneric(const std::string& name) : LinearAlgebraDense(name) {} Scalar dot(const Vector&, const Vector&) const override; void gemv(const Matrix&, const Vector&, Vector&) const override; void gemm(const Matrix&, const Matrix&, Matrix&) const override; void print(std::ostream&) const override; }; } // namespace eckit::linalg::dense eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraEigen.cc0000664000175000017500000000422215161702250023203 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/dense/LinearAlgebraEigen.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/Vector.h" #include "eckit/maths/Eigen.h" namespace eckit::linalg::dense { static const LinearAlgebraEigen __la("eigen"); using vec_t = Eigen::VectorXd::MapType; using mat_t = Eigen::MatrixXd::MapType; void LinearAlgebraEigen::print(std::ostream& out) const { out << "LinearAlgebraEigen[]"; } Scalar LinearAlgebraEigen::dot(const Vector& x, const Vector& y) const { ASSERT(x.size() == y.size()); // Eigen requires non-const pointers to the data vec_t xi(Eigen::VectorXd::Map(const_cast(x.data()), x.size())); vec_t yi(Eigen::VectorXd::Map(const_cast(y.data()), y.size())); return xi.dot(yi); } void LinearAlgebraEigen::gemv(const Matrix& A, const Vector& x, Vector& y) const { ASSERT(x.size() == A.cols()); ASSERT(y.size() == A.rows()); // Eigen requires non-const pointers to the data mat_t Ai(Eigen::MatrixXd::Map(const_cast(A.data()), A.rows(), A.cols())); vec_t xi(Eigen::VectorXd::Map(const_cast(x.data()), x.size())); vec_t yi(Eigen::VectorXd::Map(y.data(), y.size())); yi = Ai * xi; } void LinearAlgebraEigen::gemm(const Matrix& A, const Matrix& B, Matrix& C) const { ASSERT(A.cols() == B.rows()); ASSERT(A.rows() == C.rows()); ASSERT(B.cols() == C.cols()); // Eigen requires non-const pointers to the data mat_t Ai(Eigen::MatrixXd::Map(const_cast(A.data()), A.rows(), A.cols())); mat_t Bi(Eigen::MatrixXd::Map(const_cast(B.data()), B.rows(), B.cols())); mat_t Ci(Eigen::MatrixXd::Map(C.data(), C.rows(), C.cols())); Ci = Ai * Bi; } } // namespace eckit::linalg::dense eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraMKL.h0000664000175000017500000000174015161702250022443 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraDense.h" namespace eckit { namespace linalg { namespace dense { struct LinearAlgebraMKL final : public LinearAlgebraDense { LinearAlgebraMKL() {} LinearAlgebraMKL(const std::string& name) : LinearAlgebraDense(name) {} Scalar dot(const Vector&, const Vector&) const override; void gemv(const Matrix&, const Vector&, Vector&) const override; void gemm(const Matrix&, const Matrix&, Matrix&) const override; void print(std::ostream&) const override; }; } // namespace dense } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraCUDA.h0000664000175000017500000000174315161702250022537 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraDense.h" namespace eckit { namespace linalg { namespace dense { struct LinearAlgebraCUDA final : public LinearAlgebraDense { LinearAlgebraCUDA() {} LinearAlgebraCUDA(const std::string& name) : LinearAlgebraDense(name) {} Scalar dot(const Vector&, const Vector&) const override; void gemv(const Matrix&, const Vector&, Vector&) const override; void gemm(const Matrix&, const Matrix&, Matrix&) const override; void print(std::ostream&) const override; }; } // namespace dense } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraEigen.h0000664000175000017500000000165715161702250023056 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraDense.h" namespace eckit::linalg::dense { struct LinearAlgebraEigen final : public LinearAlgebraDense { LinearAlgebraEigen() {} LinearAlgebraEigen(const std::string& name) : LinearAlgebraDense(name) {} Scalar dot(const Vector&, const Vector&) const override; void gemv(const Matrix&, const Vector&, Vector&) const override; void gemm(const Matrix&, const Matrix&, Matrix&) const override; void print(std::ostream&) const override; }; } // namespace eckit::linalg::dense eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraTorch.cc0000664000175000017500000000510015161702250023227 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/dense/LinearAlgebraTorch.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/Vector.h" #include "eckit/linalg/detail/Torch.h" namespace eckit::linalg::dense { static const LinearAlgebraTorch LA_TORCH_CPU_1("torch", torch::DeviceType::CPU); static const LinearAlgebraTorch LA_TORCH_CPU_2("torch-cpu", torch::DeviceType::CPU); static const LinearAlgebraTorch LA_TORCH_CUDA("torch-cuda", torch::DeviceType::CUDA); static const LinearAlgebraTorch LA_TORCH_HIP("torch-hip", torch::DeviceType::HIP); static const LinearAlgebraTorch LA_TORCH_MPS("torch-mps", torch::DeviceType::MPS, torch::kFloat32); static const LinearAlgebraTorch LA_TORCH_XPU("torch-xpu", torch::DeviceType::XPU); static const LinearAlgebraTorch LA_TORCH_XLA("torch-xla", torch::DeviceType::XLA); static const LinearAlgebraTorch LA_TORCH_META("torch-meta", torch::DeviceType::Meta); Scalar LinearAlgebraTorch::dot(const Vector& x, const Vector& y) const { ASSERT(x.size() == y.size()); auto x_tensor = make_dense_tensor(x); auto y_tensor = make_dense_tensor(y); return tensor_to_host(torch::dot(x_tensor, y_tensor)).item(); } void LinearAlgebraTorch::gemv(const Matrix& A, const Vector& x, Vector& y) const { ASSERT(A.cols() == x.rows()); ASSERT(A.rows() == y.rows()); // multiplication auto A_tensor = make_dense_tensor(A); auto x_tensor = make_dense_tensor(x); auto y_tensor = tensor_to_host(torch::matmul(A_tensor, x_tensor)); // assignment std::memcpy(y.data(), y_tensor.data_ptr(), y.rows() * sizeof(Scalar)); } void LinearAlgebraTorch::gemm(const Matrix& A, const Matrix& X, Matrix& Y) const { ASSERT(A.cols() == X.rows()); ASSERT(A.rows() == Y.rows()); ASSERT(X.cols() == Y.cols()); // multiplication and conversion from column-major to row-major (and back) auto A_tensor = make_dense_tensor(A); auto X_tensor = make_dense_tensor(X); auto Y_tensor = tensor_transpose(tensor_to_host(torch::matmul(A_tensor, X_tensor))); // assignment std::memcpy(Y.data(), Y_tensor.data_ptr(), Y.size() * sizeof(Scalar)); } } // namespace eckit::linalg::dense eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraCUDA.cc0000664000175000017500000001174615161702250022701 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/dense/LinearAlgebraCUDA.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/Vector.h" #include "eckit/linalg/detail/CUDA.h" namespace eckit { namespace linalg { namespace dense { static const LinearAlgebraCUDA __la("cuda"); void LinearAlgebraCUDA::print(std::ostream& out) const { out << "LinearAlgebraCUDA[]"; } Scalar LinearAlgebraCUDA::dot(const Vector& x, const Vector& y) const { ASSERT(x.size() == y.size()); const auto size = Size(x.size() * sizeof(Scalar)); Scalar r; Scalar* d_x; ///< device memory vector x Scalar* d_y; ///< device memory vector y cublasHandle_t handle; CALL_CUDA(cudaMalloc((void**)&d_x, size)); CALL_CUDA(cudaMalloc((void**)&d_y, size)); CALL_CUBLAS(cublasCreate(&handle)); CALL_CUDA(cudaMemcpy(d_x, x.data(), size, cudaMemcpyHostToDevice)); CALL_CUDA(cudaMemcpy(d_y, y.data(), size, cudaMemcpyHostToDevice)); // cublasStatus_t cublasDdot (cublasHandle_t handle, int n, // const double *x, int incx, // const double *y, int incy, // double *result) CALL_CUBLAS(cublasDdot(handle, x.size(), d_x, 1, d_y, 1, &r)); CALL_CUBLAS(cublasDestroy(handle)); CALL_CUDA(cudaFree(d_x)); CALL_CUDA(cudaFree(d_y)); return r; } void LinearAlgebraCUDA::gemv(const Matrix& A, const Vector& x, Vector& y) const { ASSERT(x.size() == A.cols()); ASSERT(y.size() == A.rows()); const auto sizeA = Size(A.rows() * A.cols() * sizeof(Scalar)); const auto sizex = Size(A.cols() * sizeof(Scalar)); const auto sizey = Size(A.rows() * sizeof(Scalar)); Scalar* d_A; ///< device memory matrix A Scalar* d_x; ///< device memory vector x Scalar* d_y; ///< device memory vector y cublasHandle_t handle; CALL_CUDA(cudaMalloc((void**)&d_A, sizeA)); CALL_CUDA(cudaMalloc((void**)&d_x, sizex)); CALL_CUDA(cudaMalloc((void**)&d_y, sizey)); CALL_CUBLAS(cublasCreate(&handle)); CALL_CUDA(cudaMemcpy(d_A, A.data(), sizeA, cudaMemcpyHostToDevice)); CALL_CUDA(cudaMemcpy(d_x, x.data(), sizex, cudaMemcpyHostToDevice)); const Scalar alpha = 1.0; const Scalar beta = 0.0; // cublasStatus_t cublasDgemv(cublasHandle_t handle, cublasOperation_t trans, // int m, int n, // const double *alpha, const double *A, int lda, const double *x, int incx, // const double *beta, double *y, int incy) CALL_CUBLAS(cublasDgemv(handle, CUBLAS_OP_N, A.rows(), A.cols(), &alpha, d_A, A.rows(), d_x, 1, &beta, d_y, 1)); CALL_CUDA(cudaMemcpy(y.data(), d_y, sizey, cudaMemcpyDeviceToHost)); CALL_CUBLAS(cublasDestroy(handle)); CALL_CUDA(cudaFree(d_A)); CALL_CUDA(cudaFree(d_x)); CALL_CUDA(cudaFree(d_y)); } void LinearAlgebraCUDA::gemm(const Matrix& A, const Matrix& B, Matrix& C) const { ASSERT(A.cols() == B.rows()); ASSERT(A.rows() == C.rows()); ASSERT(B.cols() == C.cols()); const auto sizeA = Size(A.rows() * A.cols() * sizeof(Scalar)); const auto sizeB = Size(B.rows() * B.cols() * sizeof(Scalar)); const auto sizeC = Size(A.rows() * B.cols() * sizeof(Scalar)); Scalar* d_A; ///< device memory matrix A Scalar* d_B; ///< device memory matrix B Scalar* d_C; ///< device memory matrix C cublasHandle_t handle; CALL_CUDA(cudaMalloc((void**)&d_A, sizeA)); CALL_CUDA(cudaMalloc((void**)&d_B, sizeB)); CALL_CUDA(cudaMalloc((void**)&d_C, sizeC)); CALL_CUBLAS(cublasCreate(&handle)); CALL_CUDA(cudaMemcpy(d_A, A.data(), sizeA, cudaMemcpyHostToDevice)); CALL_CUDA(cudaMemcpy(d_B, B.data(), sizeB, cudaMemcpyHostToDevice)); const Scalar alpha = 1.0; const Scalar beta = 0.0; // cublasStatus_t cublasDgemm(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, // int m, int n, int k, // const double *alpha, const double *A, int lda, const double *B, int ldb, // const double *beta, double *C, int ldc) CALL_CUBLAS(cublasDgemm(handle, CUBLAS_OP_N, CUBLAS_OP_N, A.rows(), B.cols(), A.cols(), &alpha, d_A, A.rows(), d_B, B.rows(), &beta, d_C, A.rows())); CALL_CUDA(cudaMemcpy(C.data(), d_C, sizeC, cudaMemcpyDeviceToHost)); CALL_CUBLAS(cublasDestroy(handle)); CALL_CUDA(cudaFree(d_A)); CALL_CUDA(cudaFree(d_B)); CALL_CUDA(cudaFree(d_C)); } } // namespace dense } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraTorch.h0000664000175000017500000000213115161702250023072 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/LinearAlgebraDense.h" #include "eckit/linalg/detail/Torch.h" namespace eckit::linalg::dense { struct LinearAlgebraTorch final : public LinearAlgebraDense, detail::Torch { LinearAlgebraTorch(const std::string& name, torch::DeviceType device, torch::ScalarType scalar = torch::kFloat64) : LinearAlgebraDense(name), Torch(device, scalar) {} Scalar dot(const Vector& x, const Vector& y) const override; void gemv(const Matrix& A, const Vector& x, Vector& y) const override; void gemm(const Matrix& A, const Matrix& X, Matrix& Y) const override; void print(std::ostream& os) const override { Torch::print(os); } }; } // namespace eckit::linalg::dense eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraMKL.cc0000664000175000017500000000651515161702250022606 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/dense/LinearAlgebraMKL.h" #include "mkl.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/Vector.h" namespace eckit { namespace linalg { namespace dense { static const LinearAlgebraMKL __la("mkl"); static const MKL_INT inc = 1; static const CBLAS_LAYOUT layout = CblasColMajor; static const CBLAS_TRANSPOSE trans = CblasNoTrans; static const double alpha = 1.; static const double beta = 0.; void LinearAlgebraMKL::print(std::ostream& out) const { out << "LinearAlgebraMKL[]"; } Scalar LinearAlgebraMKL::dot(const Vector& x, const Vector& y) const { ASSERT(x.size() == y.size()); const auto n = static_cast(x.size()); const auto* _x = static_cast(x.data()); const auto* _y = static_cast(y.data()); // double cblas_ddot(const MKL_INT N, const double *X, const MKL_INT incX, // const double *Y, const MKL_INT incY); return cblas_ddot(n, _x, inc, _y, inc); } void LinearAlgebraMKL::gemv(const Matrix& A, const Vector& x, Vector& y) const { ASSERT(x.size() == A.cols()); ASSERT(y.size() == A.rows()); const auto m = static_cast(A.rows()); const auto n = static_cast(A.cols()); const auto* _A = static_cast(A.data()); const auto* _x = static_cast(x.data()); auto* _y = static_cast(y.data()); // void cblas_dgemv(const CBLAS_LAYOUT Layout, // const CBLAS_TRANSPOSE TransA, const MKL_INT M, const MKL_INT N, // const double alpha, const double *A, const MKL_INT lda, // const double *X, const MKL_INT incX, const double beta, // double *Y, const MKL_INT incY); cblas_dgemv(layout, trans, m, n, alpha, _A, m, _x, inc, beta, _y, inc); } void LinearAlgebraMKL::gemm(const Matrix& A, const Matrix& B, Matrix& C) const { ASSERT(A.cols() == B.rows()); ASSERT(A.rows() == C.rows()); ASSERT(B.cols() == C.cols()); const auto m = static_cast(A.rows()); const auto n = static_cast(B.cols()); const auto k = static_cast(A.cols()); const auto* _A = static_cast(A.data()); const auto* _B = static_cast(B.data()); auto* _C = static_cast(C.data()); // void cblas_dgemm(const CBLAS_LAYOUT Layout, const CBLAS_TRANSPOSE TransA, // const CBLAS_TRANSPOSE TransB, const MKL_INT M, const MKL_INT N, // const MKL_INT K, const double alpha, const double *A, // const MKL_INT lda, const double *B, const MKL_INT ldb, // const double beta, double *C, const MKL_INT ldc); cblas_dgemm(layout, trans, trans, m, n, k, alpha, _A, m, _B, k, beta, _C, m); } } // namespace dense } // namespace linalg } // namespace eckit eckit-2.0.7/src/eckit/linalg/dense/LinearAlgebraGeneric.cc0000664000175000017500000000436515161702250023540 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/dense/LinearAlgebraGeneric.h" #include #include #include "eckit/eckit_config.h" #include "eckit/exception/Exceptions.h" #include "eckit/linalg/Matrix.h" #include "eckit/linalg/Vector.h" namespace eckit::linalg::dense { static const LinearAlgebraGeneric __la_generic("generic"); #if eckit_HAVE_OMP static const LinearAlgebraGeneric __la_openmp("openmp"); #endif void LinearAlgebraGeneric::print(std::ostream& out) const { out << "LinearAlgebraGeneric[]"; } Scalar LinearAlgebraGeneric::dot(const Vector& x, const Vector& y) const { const auto Ni = x.size(); ASSERT(y.size() == Ni); Scalar sum = 0.; #if eckit_HAVE_OMP #pragma omp parallel for reduction(+ : sum) #endif for (Size i = 0; i < Ni; ++i) { const auto p = x[i] * y[i]; sum += p; } return sum; } void LinearAlgebraGeneric::gemv(const Matrix& A, const Vector& x, Vector& y) const { const auto Ni = A.rows(); const auto Nj = A.cols(); ASSERT(y.rows() == Ni); ASSERT(x.rows() == Nj); #if eckit_HAVE_OMP #pragma omp parallel for #endif for (Size i = 0; i < Ni; ++i) { Scalar sum = 0.; for (Size j = 0; j < Nj; ++j) { sum += A(i, j) * x[j]; } y[i] = sum; } } void LinearAlgebraGeneric::gemm(const Matrix& A, const Matrix& B, Matrix& C) const { const auto Ni = A.rows(); const auto Nj = B.cols(); const auto Nk = A.cols(); ASSERT(C.rows() == Ni); ASSERT(C.cols() == Nj); ASSERT(B.rows() == Nk); #if eckit_HAVE_OMP #pragma omp parallel for collapse(2) #endif for (Size j = 0; j < Nj; ++j) { for (Size i = 0; i < Ni; ++i) { Scalar sum = 0.; for (Size k = 0; k < Nk; ++k) { sum += A(i, k) * B(k, j); } C(i, j) = sum; } } } } // namespace eckit::linalg::dense eckit-2.0.7/src/eckit/linalg/detail/0000775000175000017500000000000015161702250017400 5ustar alastairalastaireckit-2.0.7/src/eckit/linalg/detail/Torch.cc0000664000175000017500000000466315161702250020777 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/detail/Torch.h" #include #include #include "eckit/linalg/Matrix.h" #include "eckit/linalg/SparseMatrix.h" #include "eckit/linalg/Vector.h" namespace eckit::linalg::detail { static_assert(std::is_same::value, "Index type mismatch"); static_assert(std::is_same::value, "Scalar type mismatch"); torch::Tensor Torch::tensor_transpose(const torch::Tensor& tensor) const { return tensor.transpose(0, 1).contiguous(); } torch::Tensor Torch::tensor_to_host(const torch::Tensor& tensor) const { return tensor.to(torch::DeviceType::CPU, torch::kFloat64).contiguous(); // reverse MPS float32 (if applicable) } torch::Tensor Torch::make_dense_tensor(const Matrix& A) const { auto Ni = static_cast(A.cols()); auto Nj = static_cast(A.rows()); return tensor_transpose( torch::from_blob(const_cast(A.data()), {Ni, Nj}, torch::kFloat64).to(device_, scalar_)); } torch::Tensor Torch::make_dense_tensor(const Vector& V) const { auto Ni = static_cast(V.size()); return torch::from_blob(const_cast(V.data()), {Ni}, torch::kFloat64).to(device_, scalar_); } torch::Tensor Torch::make_sparse_csr_tensor(const SparseMatrix& A) const { auto Ni = static_cast(A.rows()); auto Nj = static_cast(A.cols()); auto Nz = static_cast(A.nonZeros()); auto ia = torch::from_blob(const_cast(A.outer()), {Ni + 1}, torch::kInt32).to(device_, torch::kInt64); auto ja = torch::from_blob(const_cast(A.inner()), {Nz}, torch::kInt32).to(device_, torch::kInt64); auto a = torch::from_blob(const_cast(A.data()), {Nz}, torch::kFloat64).to(device_, scalar_); return torch::sparse_csr_tensor(ia, ja, a, {Ni, Nj}, torch::TensorOptions().dtype(scalar_).device(device_).layout(torch::kSparseCsr)); } void Torch::print(std::ostream& os) const { os << "LinearAlgebraTorch[device=" << device_ << "]"; } } // namespace eckit::linalg::detail eckit-2.0.7/src/eckit/linalg/detail/Torch.h0000664000175000017500000000262315161702250020633 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include "eckit/linalg/types.h" #include "torch/torch.h" namespace eckit::linalg::detail { /** * @brief Torch tensor creation and device management for linear algebra backends. * * Copies data host to/from device per operation. Transfer overhead may outweigh accelerator device gains for * small/frequent operations; best suited for large matrices where compute dominates. */ class Torch { protected: explicit Torch(torch::DeviceType device, torch::ScalarType scalar) : device_(device), scalar_(scalar) {} torch::Tensor tensor_transpose(const torch::Tensor&) const; torch::Tensor tensor_to_host(const torch::Tensor&) const; torch::Tensor make_dense_tensor(const Matrix&) const; torch::Tensor make_dense_tensor(const Vector&) const; torch::Tensor make_sparse_csr_tensor(const SparseMatrix&) const; void print(std::ostream&) const; private: const torch::DeviceType device_; const torch::ScalarType scalar_; }; } // namespace eckit::linalg::detail eckit-2.0.7/src/eckit/linalg/detail/HIP.h0000664000175000017500000000357315161702250020201 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ // There is a name clash because hip_runtime defines DEPRECATED, and eckit as well #ifdef DEPRECATED #undef DEPRECATED #endif #include #include #include // There is a name clash because hip_runtime defines DEPRECATED, and eckit as well #ifdef DEPRECATED #undef DEPRECATED #endif #define CALL_HIP(e) \ { \ hipError_t error; \ if ((error = e) != hipSuccess) \ printf("%s failed with error code %d @ %s +%d\n", #e, error, __FILE__, __LINE__), exit(EXIT_FAILURE); \ } #define CALL_HIPSPARSE(e) \ { \ hipsparseStatus_t error; \ if ((error = e) != HIPSPARSE_STATUS_SUCCESS) \ printf("%s failed with error code %d @ %s +%d\n", #e, error, __FILE__, __LINE__), exit(EXIT_FAILURE); \ } eckit-2.0.7/src/eckit/linalg/detail/CUDA.h0000664000175000017500000000426215161702250020271 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include #include #include #define CALL_CUDA(e) \ { \ cudaError_t error; \ if ((error = e) != cudaSuccess) \ printf("%s failed with error code %d @ %s +%d\n", #e, error, __FILE__, __LINE__), exit(EXIT_FAILURE); \ } #define CALL_CUBLAS(e) \ { \ cublasStatus_t error; \ if ((error = e) != CUBLAS_STATUS_SUCCESS) \ printf("%s failed with error code %d @ %s +%d\n", #e, error, __FILE__, __LINE__), exit(EXIT_FAILURE); \ } #define CALL_CUSPARSE(e) \ { \ cusparseStatus_t error; \ if ((error = e) != CUSPARSE_STATUS_SUCCESS) \ printf("%s failed with error code %d @ %s +%d\n", #e, error, __FILE__, __LINE__), exit(EXIT_FAILURE); \ } eckit-2.0.7/src/eckit/linalg/Matrix.cc0000664000175000017500000000574315161702250017722 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/Matrix.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/serialisation/Stream.h" namespace eckit::linalg { //---------------------------------------------------------------------------------------------------------------------- Matrix::Matrix() : array_{nullptr}, rows_(0), cols_(0), own_(false) {} Matrix::Matrix(Size rows, Size cols) : array_(new Scalar[rows * cols]), rows_(rows), cols_(cols), own_(true) { ASSERT(size() > 0); ASSERT(array_); } Matrix::Matrix(const Scalar* array, Size rows, Size cols) : array_(const_cast(array)), rows_(rows), cols_(cols), own_(false) { ASSERT(size() > 0); ASSERT(array_); } Matrix::Matrix(Stream& stream) : array_{nullptr}, rows_(0), cols_(0), own_(false) { Size rows, cols; stream >> rows; stream >> cols; resize(rows, cols); ASSERT(size() > 0); ASSERT(array_); stream.readBlob(array_, (rows * cols) * sizeof(Scalar)); } Matrix::Matrix(const Matrix& other) : array_(new Scalar[other.size()]), rows_(other.rows_), cols_(other.cols_), own_(true) { ASSERT(size() > 0); ASSERT(array_); ::memcpy(array_, other.array_, size() * sizeof(Scalar)); } Matrix::~Matrix() { if (own_) { delete[] array_; } } Matrix& Matrix::operator=(const Matrix& other) { // do not optimize for if size()==other.size(), as using copy constructor // consistently retains ownership (no surprises in ownership behaviour) Matrix copy(other); swap(copy); return *this; } void Matrix::swap(Matrix& other) { std::swap(array_, other.array_); std::swap(rows_, other.rows_); std::swap(cols_, other.cols_); std::swap(own_, other.own_); } void Matrix::resize(Size rows, Size cols) { // avoid reallocation if memory is the same if (size() != rows * cols) { Matrix m(rows, cols); swap(m); } rows_ = rows; cols_ = cols; } void Matrix::setZero() { ASSERT(size() > 0); ASSERT(array_); ::memset(array_, 0, size() * sizeof(Scalar)); } void Matrix::fill(Scalar value) { for (Size i = 0; i < size(); ++i) { array_[i] = value; } } void Matrix::encode(Stream& stream) const { stream << rows_; stream << cols_; stream.writeBlob(const_cast(array_), rows_ * cols_ * sizeof(Scalar)); } Stream& operator<<(Stream& stream, const Matrix& matrix) { matrix.encode(stream); return stream; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/BackendRegistry.h0000664000175000017500000000466215161702250021377 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" namespace eckit::linalg { template class BackendRegistry { public: BackendRegistry(const char* default_name, const char* env_var) : default_(default_name) { ASSERT(!default_.empty()); const auto* envBackend = ::getenv(env_var); if (envBackend != nullptr) { default_ = envBackend; } } void backend(const std::string& name) { AutoLock lock(mutex_); if (map_.find(name) == map_.end()) { throw BadParameter("Invalid backend " + name, Here()); } ASSERT(!name.empty()); default_ = name; } const std::string& name() const { return default_; } bool has(const std::string& name) const { AutoLock lock(mutex_); return map_.find(name) != map_.end(); } const LA& find(const std::string& name = "") const { AutoLock lock(mutex_); if (name.empty()) { ASSERT(!default_.empty()); return find(default_); } auto it = map_.find(name); if (it == map_.end()) { auto msg = "No backend named [" + name + "]."; list(Log::error() << msg + " Backends are: ") << std::endl; throw BadParameter(msg, Here()); } return *(it->second); } std::ostream& list(std::ostream& out) const { AutoLock lock(mutex_); const auto* sep = ""; for (auto backend : map_) { out << sep << backend.first; sep = ", "; } return out; } void add(const std::string& name, LA* backend) { AutoLock lock(mutex_); ASSERT(map_.find(name) == map_.end()); map_[name] = backend; } private: // members std::map map_; std::string default_; mutable Mutex mutex_; }; } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/allocator/0000775000175000017500000000000015161702250020116 5ustar alastairalastaireckit-2.0.7/src/eckit/linalg/allocator/BufferAllocator.h0000664000175000017500000000165215161702250023345 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/SparseMatrix.h" #include "eckit/memory/MemoryBuffer.h" namespace eckit::linalg::allocator { class BufferAllocator : public SparseMatrix::Allocator { public: BufferAllocator(const MemoryBuffer&); SparseMatrix::Layout allocate(SparseMatrix::Shape&) override; void deallocate(SparseMatrix::Layout, SparseMatrix::Shape) override; bool inSharedMemory() const override; void print(std::ostream&) const override; MemoryBuffer buffer_; }; } // namespace eckit::linalg::allocator eckit-2.0.7/src/eckit/linalg/allocator/StandardContainerAllocator.cc0000664000175000017500000000406015161702250025671 0ustar alastairalastair/* * (C) Copyright 2024- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/allocator/StandardContainerAllocator.h" #include #include #include "eckit/exception/Exceptions.h" namespace eckit::linalg::allocator { StandardContainerAllocator::StandardContainerAllocator(Size Nr, Size Nc, const container_type& cont) : Nr_(Nr), Nc_(Nc), nnz_([Nc](const auto& cont) { return std::accumulate(cont.begin(), cont.end(), 0, [Nc](const auto& acc, const auto& row) { ASSERT(row.empty() || row.rbegin()->first < Nc); return acc + row.size(); }); }(cont)) { ASSERT(Nr == cont.size()); ia_.reserve(Nr_ + 1); ja_.reserve(nnz_); a_.reserve(nnz_); ia_.emplace_back(0); for (const auto& row : cont) { ia_.emplace_back(ia_.back() + row.size()); for (const auto& [col, val] : row) { ja_.emplace_back(static_cast(col)); a_.emplace_back(val); } } } SparseMatrix::Layout StandardContainerAllocator::StandardContainerAllocator::allocate(SparseMatrix::Shape& shape) { shape.size_ = nnz_; shape.rows_ = Nr_; shape.cols_ = Nc_; SparseMatrix::Layout layout; layout.outer_ = ia_.data(); layout.inner_ = ja_.data(); layout.data_ = a_.data(); return layout; } void StandardContainerAllocator::deallocate(SparseMatrix::Layout, SparseMatrix::Shape) { ia_.clear(); ja_.clear(); a_.clear(); } bool StandardContainerAllocator::inSharedMemory() const { return false; } void StandardContainerAllocator::print(std::ostream& out) const { out << "StandardContainerAllocator[Nr=" << Nr_ << ",Nc=" << Nc_ << ",nnz=" << nnz_ << "]"; } } // namespace eckit::linalg::allocator eckit-2.0.7/src/eckit/linalg/allocator/NonOwningAllocator.h0000664000175000017500000000235615161702250024052 0ustar alastairalastair/* * (C) Copyright 2024- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/SparseMatrix.h" namespace eckit::linalg::allocator { /** * @brief In-place allocator for sparse matrices, directly mapping supporting arrays including from another * SparseMatrix. It is able to provide a "view". */ class NonOwningAllocator : public SparseMatrix::Allocator { public: NonOwningAllocator(Size Nr, Size Nc, Size nnz, Index* ia, Index* ja, Scalar* a); SparseMatrix::Layout allocate(SparseMatrix::Shape&) override; void deallocate(SparseMatrix::Layout, SparseMatrix::Shape) override; bool inSharedMemory() const override; void print(std::ostream&) const override; private: const Size Nr_; const Size Nc_; const Size nnz_; Index* ia_; // NOTE: not owned Index* ja_; // NOTE: not owned Scalar* a_; // NOTE: not owned }; } // namespace eckit::linalg::allocator eckit-2.0.7/src/eckit/linalg/allocator/NonOwningAllocator.cc0000664000175000017500000000274115161702250024206 0ustar alastairalastair/* * (C) Copyright 2024- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/allocator/NonOwningAllocator.h" #include #include "eckit/exception/Exceptions.h" namespace eckit::linalg::allocator { NonOwningAllocator::NonOwningAllocator(Size Nr, Size Nc, Size nnz, Index* ia, Index* ja, Scalar* a) : Nr_(Nr), Nc_(Nc), nnz_(nnz), ia_(ia), ja_(ja), a_(a) { ASSERT(ia_ != nullptr); ASSERT(ja_ != nullptr); ASSERT(a_ != nullptr); } SparseMatrix::Layout NonOwningAllocator::NonOwningAllocator::allocate(SparseMatrix::Shape& shape) { shape.size_ = nnz_; shape.rows_ = Nr_; shape.cols_ = Nc_; SparseMatrix::Layout layout; layout.outer_ = reinterpret_cast(ia_); layout.inner_ = ja_; layout.data_ = a_; return layout; } void NonOwningAllocator::deallocate(SparseMatrix::Layout, SparseMatrix::Shape) {} bool NonOwningAllocator::inSharedMemory() const { return false; } void NonOwningAllocator::print(std::ostream& out) const { out << "NonOwningAllocator[Nr=" << Nr_ << ",Nc=" << Nc_ << ",nnz=" << nnz_ << "]"; } } // namespace eckit::linalg::allocator eckit-2.0.7/src/eckit/linalg/allocator/StandardContainerAllocator.h0000664000175000017500000000250515161702250025535 0ustar alastairalastair/* * (C) Copyright 2024- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/linalg/SparseMatrix.h" namespace eckit::linalg::allocator { /** * @brief In-place allocator for sparse matrices, directly mapping supporting arrays including from another * SparseMatrix. It is able to provide a "view". */ class StandardContainerAllocator : public SparseMatrix::Allocator { public: using container_type = std::vector>; StandardContainerAllocator(Size Nr, Size Nc, const container_type&); SparseMatrix::Layout allocate(SparseMatrix::Shape&) override; void deallocate(SparseMatrix::Layout, SparseMatrix::Shape) override; bool inSharedMemory() const override; void print(std::ostream&) const override; private: const Size Nr_; const Size Nc_; const Size nnz_; std::vector ia_; std::vector ja_; std::vector a_; }; } // namespace eckit::linalg::allocator eckit-2.0.7/src/eckit/linalg/allocator/StandardAllocator.h0000664000175000017500000000163315161702250023673 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #pragma once #include "eckit/linalg/SparseMatrix.h" #include "eckit/memory/MemoryBuffer.h" namespace eckit::linalg::allocator { class StandardAllocator : public SparseMatrix::Allocator { public: StandardAllocator(); SparseMatrix::Layout allocate(SparseMatrix::Shape&) override; void deallocate(SparseMatrix::Layout, SparseMatrix::Shape) override; bool inSharedMemory() const override; void print(std::ostream&) const override; MemoryBuffer buffer_; }; } // namespace eckit::linalg::allocator eckit-2.0.7/src/eckit/linalg/allocator/StandardAllocator.cc0000664000175000017500000000263515161702250024034 0ustar alastairalastair /* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/allocator/StandardAllocator.h" #include #include "eckit/log/Bytes.h" namespace eckit::linalg::allocator { StandardAllocator::StandardAllocator() : buffer_(0) {} SparseMatrix::Layout StandardAllocator::allocate(SparseMatrix::Shape& shape) { if (shape.allocSize() > buffer_.size()) { buffer_.resize(shape.allocSize()); } SparseMatrix::Layout layout; char* addr = buffer_; layout.data_ = reinterpret_cast(addr); layout.outer_ = reinterpret_cast(addr + shape.sizeofData()); layout.inner_ = reinterpret_cast(addr + shape.sizeofData() + shape.sizeofOuter()); return layout; } void StandardAllocator::deallocate(SparseMatrix::Layout p, SparseMatrix::Shape) {} bool StandardAllocator::inSharedMemory() const { return false; } void StandardAllocator::print(std::ostream& out) const { out << "StandardAllocator[" << Bytes{static_cast(buffer_.size())} << "]"; } } // namespace eckit::linalg::allocator eckit-2.0.7/src/eckit/linalg/allocator/BufferAllocator.cc0000664000175000017500000000222615161702250023501 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/allocator/BufferAllocator.h" #include #include "eckit/log/Bytes.h" namespace eckit::linalg::allocator { BufferAllocator::BufferAllocator(const MemoryBuffer& buffer) : buffer_(buffer, buffer.size()) {} SparseMatrix::Layout BufferAllocator::allocate(SparseMatrix::Shape& shape) { SparseMatrix::Layout layout; SparseMatrix::load(buffer_.data(), buffer_.size(), layout, shape); return layout; } void BufferAllocator::deallocate(SparseMatrix::Layout, SparseMatrix::Shape) {} bool BufferAllocator::inSharedMemory() const { return false; } void BufferAllocator::print(std::ostream& out) const { out << "BufferAllocator[" << Bytes{static_cast(buffer_.size())} << "]"; } } // namespace eckit::linalg::allocator eckit-2.0.7/src/eckit/linalg/types.h0000664000175000017500000000125415161702250017455 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ /// @author Florian Rathgeber /// @author Pedro Maciel /// @author Tiago Quintino #pragma once #include namespace eckit::linalg { using Scalar = double; using Index = int; using Size = size_t; class Vector; class Matrix; class SparseMatrix; } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/Triplet.h0000664000175000017500000000302415161702250017731 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ /// @author Florian Rathgeber /// @author Pedro Maciel #pragma once #include #include "eckit/linalg/types.h" namespace eckit::linalg { //---------------------------------------------------------------------------------------------------------------------- /// Triplet of values compatible to Eigen::Triplet class Triplet { public: Triplet() : row_(0), col_(0), val_(0.) {} Triplet(const Size& i, const Size& j, const Scalar& v = {0}) : row_(i), col_(j), val_(v) {} const Size& row() const { return row_; } const Size& col() const { return col_; } const Scalar& value() const { return val_; } Scalar& value() { return val_; } bool operator<(const Triplet& other) const { return row_ == other.row_ ? col_ < other.col_ : row_ < other.row_; } bool nonZero() const { return val_ != 0.; } void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream&, const Triplet&); protected: Size row_; Size col_; Scalar val_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/Vector.cc0000664000175000017500000000453615161702250017717 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include "eckit/linalg/Vector.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/serialisation/Stream.h" namespace eckit::linalg { //---------------------------------------------------------------------------------------------------------------------- Vector::Vector() : array_{nullptr}, length_(0), own_(false) {} Vector::Vector(Size length) : array_(new Scalar[length]), length_(length), own_(true) {} Vector::Vector(const Scalar array[], Size length) : array_(const_cast(array)), length_(length), own_(false) { ASSERT(array_ && length_ > 0); } Vector::Vector(Stream& stream) : array_{nullptr}, length_(0), own_(false) { Size length; stream >> length; resize(length); ASSERT(length_ > 0); stream.readBlob(array_, length * sizeof(Scalar)); } Vector::Vector(const Vector& other) : array_(new Scalar[other.length_]), length_(other.length_), own_(true) { ::memcpy(array_, other.array_, length_ * sizeof(Scalar)); } Vector::~Vector() { if (own_) { delete[] array_; } } Vector& Vector::operator=(const Vector& other) { Vector copy(other); swap(copy); return *this; } void Vector::swap(Vector& other) { std::swap(array_, other.array_); std::swap(length_, other.length_); std::swap(own_, other.own_); } void Vector::resize(Size length) { Vector v(length); swap(v); } void Vector::setZero() { ::memset(array_, 0, length_ * sizeof(Scalar)); } void Vector::fill(Scalar value) { for (Size i = 0; i < length_; ++i) { array_[i] = value; } } void Vector::encode(Stream& stream) const { stream << length_; stream.writeBlob(array_, length_ * sizeof(Scalar)); } Stream& operator<<(Stream& stream, const Vector& vector) { vector.encode(stream); return stream; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::linalg eckit-2.0.7/src/eckit/linalg/CMakeLists.txt0000664000175000017500000000643315161702250020704 0ustar alastairalastairunset( eckit_la_pincludes ) unset( eckit_la_plibs ) list( APPEND eckit_la_srcs BackendRegistry.h LinearAlgebra.cc LinearAlgebra.h LinearAlgebraDense.cc LinearAlgebraDense.h LinearAlgebraSparse.cc LinearAlgebraSparse.h Matrix.cc Matrix.h SparseMatrix.cc SparseMatrix.h Tensor.cc Tensor.h Triplet.cc Triplet.h Vector.cc Vector.h allocator/BufferAllocator.cc allocator/BufferAllocator.h allocator/NonOwningAllocator.cc allocator/NonOwningAllocator.h allocator/StandardAllocator.cc allocator/StandardAllocator.h allocator/StandardContainerAllocator.cc allocator/StandardContainerAllocator.h dense/LinearAlgebraGeneric.cc dense/LinearAlgebraGeneric.h sparse/LinearAlgebraGeneric.cc sparse/LinearAlgebraGeneric.h types.h ) if( eckit_HAVE_CUDA ) list( APPEND eckit_la_srcs dense/LinearAlgebraCUDA.cc dense/LinearAlgebraCUDA.h detail/CUDA.h sparse/LinearAlgebraCUDA.cc sparse/LinearAlgebraCUDA.h ) list( APPEND eckit_la_plibs CUDA::cudart CUDA::cusparse CUDA::cublas ) endif() if( eckit_HAVE_HIP ) list( APPEND eckit_la_srcs sparse/LinearAlgebraHIP.cc sparse/LinearAlgebraHIP.h ) list( APPEND eckit_la_plibs hip::host roc::hipsparse ) endif() if( eckit_HAVE_EIGEN ) list( APPEND eckit_la_srcs dense/LinearAlgebraEigen.cc dense/LinearAlgebraEigen.h sparse/LinearAlgebraEigen.cc sparse/LinearAlgebraEigen.h ) endif() if( eckit_HAVE_LAPACK ) list( APPEND eckit_la_srcs dense/LinearAlgebraLAPACK.cc dense/LinearAlgebraLAPACK.h ) list( APPEND eckit_la_plibs "${LAPACK_LIBRARIES}" ) endif() if( eckit_HAVE_MKL ) list( APPEND eckit_la_srcs dense/LinearAlgebraMKL.cc dense/LinearAlgebraMKL.h sparse/LinearAlgebraMKL.cc sparse/LinearAlgebraMKL.h ) list( APPEND eckit_la_pincludes "${MKL_INCLUDE_DIRS}" ) list( APPEND eckit_la_plibs "${MKL_LIBRARIES}" ) endif() if( eckit_HAVE_TORCH ) list( APPEND eckit_la_srcs dense/LinearAlgebraTorch.cc dense/LinearAlgebraTorch.h detail/Torch.cc detail/Torch.h sparse/LinearAlgebraTorch.cc sparse/LinearAlgebraTorch.h ) list( APPEND eckit_la_pincludes torch ) list( APPEND eckit_la_plibs torch ) endif() if( eckit_HAVE_OMP ) list( APPEND eckit_la_plibs OpenMP::OpenMP_CXX ) endif() ecbuild_add_library( TARGET eckit_linalg TYPE SHARED INSTALL_HEADERS ALL HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/linalg SOURCES ${eckit_la_srcs} PRIVATE_INCLUDES ${eckit_la_pincludes} PUBLIC_LIBS eckit PRIVATE_LIBS ${eckit_la_plibs} ) if( eckit_HAVE_EIGEN ) if(NOT EIGEN3_INCLUDE_DIRS AND TARGET Eigen3::Eigen) get_target_property(EIGEN3_INCLUDE_DIRS Eigen3::Eigen INTERFACE_INCLUDE_DIRECTORIES) endif() # Add include directories with "SYSTEM" to avoid warnings from within Eigen headers target_include_directories( eckit_linalg SYSTEM PRIVATE ${EIGEN3_INCLUDE_DIRS} ) endif() eckit-2.0.7/src/eckit/maths/0000775000175000017500000000000015161702250016004 5ustar alastairalastaireckit-2.0.7/src/eckit/maths/FloatingPointExceptions.cc0000664000175000017500000002515615161702250023143 0ustar alastairalastair/* * (C) Copyright 2013- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/maths/FloatingPointExceptions.h" #include #include #include #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/eckit_config.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/Translator.h" #define ECKIT_SUPPORT_STDC_FENV_ACCESS_ON 1 #ifdef __clang__ #if __clang_major__ < 17 #warning We are not using '#pragma STDC FENV_ACCESS ON' when the compiler identifies as 'clang version < 17'. This is because AMD aocc version < 5 crashes with this pragma but cannot be discerned from clang with compile time checks. (see https://github.com/ecmwf/eckit/pull/206)" #undef ECKIT_SUPPORT_STDC_FENV_ACCESS_ON #define ECKIT_SUPPORT_STDC_FENV_ACCESS_ON 0 #endif #endif #if ECKIT_SUPPORT_STDC_FENV_ACCESS_ON #pragma STDC FENV_ACCESS ON #endif namespace eckit::maths { //---------------------------------------------------------------------------------------------------------------------- namespace { int make_excepts(const std::string& codes) { static const std::map CODE_TO_EXCEPT{ {"FE_INVALID", FE_INVALID}, {"FE_INEXACT", FE_INEXACT}, {"FE_DIVBYZERO", FE_DIVBYZERO}, {"FE_OVERFLOW", FE_OVERFLOW}, {"FE_UNDERFLOW", FE_UNDERFLOW}, }; int excepts = 0; for (const auto& code : translate>(codes)) { if (code == "FE_ALL_EXCEPT") { excepts |= FE_ALL_EXCEPT; break; } if (auto se = CODE_TO_EXCEPT.find(code); se != CODE_TO_EXCEPT.end()) { excepts |= se->second & FE_ALL_EXCEPT; continue; } std::vector valid_codes; for (const auto& [c, e] : CODE_TO_EXCEPT) { valid_codes.push_back(c); } throw UserError("FloatingPointExceptions: '" + code + "' is not valid (" + translate(valid_codes) + ",FE_ALL_EXCEPT)", Here()); } return excepts; } [[noreturn]] void custom_signal_handler(int signum, ::siginfo_t* si, [[maybe_unused]] void* ucontext); // TODO use/improve eckit::SignalHandler struct : private std::map> { void set( int signum, const struct ::sigaction& act = []() { struct ::sigaction sa{}; sigemptyset(&sa.sa_mask); sa.sa_sigaction = &custom_signal_handler; sa.sa_flags = SA_SIGINFO; return sa; }()) { if (auto it = find(signum); it != end()) { unset(signum); } auto& [a, b] = operator[](signum); ::sigaction(signum, &(a = act), &b); } void unset(int signum) { if (auto it = find(signum); it != end()) { ::sigaction(signum, &it->second.second, nullptr); erase(it); return; } throw UserError("FloatingPointExceptions: signal " + std::to_string(signum) + " is not set", Here()); } void unset() { while (!empty()) { unset(rbegin()->first); } } } CUSTOM_SIGNAL_HANDLERS; int CUSTOM_EXCEPTS = 0; void enable_floating_point_exception(int excepts) { #if eckit_HAVE_FEENABLEEXCEPT feenableexcept(excepts); CUSTOM_EXCEPTS |= excepts; #elif defined(__linux__) && (defined(__i386__) || defined(__x86_64__)) std::fenv_t fenv; if (std::fegetenv(&fenv) != 0) { return; } fenv.__control_word &= ~excepts; // Unmask exceptions in x87 FPU control word (clear bits) fenv.__mxcsr &= ~(excepts << 7); // Unmask exceptions in SSE MXCSR (clear bits 7-12) std::fesetenv(&fenv); CUSTOM_EXCEPTS |= excepts; #elif defined(__APPLE__) std::fenv_t fenv; if (std::fegetenv(&fenv) != 0) { return; } #if defined(__arm64__) fenv.__fpsr |= excepts; // Unmask exceptions in FPSR (set bits) fenv.__fpcr |= (excepts << 8); // Unmask exceptions in FPCR (set bits 8-15) #else fenv.__control &= ~excepts; // Unmask exceptions in x87 FPU control word (clear bits) fenv.__mxcsr &= ~(excepts << 7); // Unmask exceptions in SSE MXCSR (clear bits 7-12) #endif std::fesetenv(&fenv); CUSTOM_EXCEPTS |= excepts; #endif } void disable_floating_point_exception(int excepts) { #if eckit_HAVE_FEDISABLEEXCEPT fedisableexcept(excepts); CUSTOM_EXCEPTS &= ~excepts; #elif defined(__linux__) && (defined(__i386__) || defined(__x86_64__)) std::fenv_t fenv; if (std::fegetenv(&fenv) != 0) { return; } fenv.__control_word |= excepts; // Mask exceptions in x87 FPU control word (set bits) fenv.__mxcsr |= (excepts << 7); // Mask exceptions in SSE MXCSR (set bits 7-12) std::fesetenv(&fenv); CUSTOM_EXCEPTS &= ~excepts; #elif defined(__APPLE__) std::fenv_t fenv; if (std::fegetenv(&fenv) != 0) { return; } #if defined(__arm64__) fenv.__fpsr &= ~excepts; // Mask exceptions in FPSR (clear bits) fenv.__fpcr &= ~(excepts << 8); // Mask exceptions in FPCR (clear bits 8-15) #else fenv.__control |= excepts; // Mask exceptions in x87 FPU control word (set bits) fenv.__mxcsr |= (excepts << 7); // Mask exceptions in SSE MXCSR (set bits 7-12) #endif std::fesetenv(&fenv); CUSTOM_EXCEPTS &= ~excepts; #else // the platform does not support enabling/disabling FPEs (void)excepts; #endif } [[noreturn]] void custom_signal_handler(int signum, ::siginfo_t* si, [[maybe_unused]] void* ucontext) { #define STR(x) #x #define LOG(x) ::write(2 /*stderr*/, x "\n", sizeof(x)) LOG("---" "\nFloatingPointExceptions"); if (signum == SIGFPE) { LOG("signal: " STR(SIGFPE)); auto raised = si->si_code & FE_ALL_EXCEPT; if (0 != (raised & FPE_INTDIV)) { LOG("code: " STR(FPE_INTDIV)); } if (0 != (raised & FPE_INTOVF)) { LOG("code: " STR(FPE_INTOVF)); } if (0 != (raised & FPE_FLTDIV)) { LOG("code: " STR(FPE_FLTDIV)); } if (0 != (raised & FPE_FLTINV)) { LOG("code: " STR(FPE_FLTINV)); } if (0 != (raised & FPE_FLTOVF)) { LOG("code: " STR(FPE_FLTOVF)); } if (0 != (raised & FPE_FLTUND)) { LOG("code: " STR(FPE_FLTUND)); } if (0 != (raised & FPE_FLTRES)) { LOG("code: " STR(FPE_FLTRES)); } if (0 != (raised & FPE_FLTSUB)) { LOG("code: " STR(FPE_FLTSUB)); } } #if defined(__APPLE__) && defined(__arm64__) else if (signum == SIGILL) { LOG("signal: " STR(SIGILL)); // On Apple Silicon a SIGFPE may be posing as a SIGILL // See: // https://developer.apple.com/forums/thread/689159?answerId=733736022 // https://developer.arm.com/documentation/ddi0595/2020-12/AArch64-Registers/ESR-EL1--Exception-Syndrome-Register--EL1- // https://developer.arm.com/documentation/ddi0595/2020-12/AArch64-Registers/ESR-EL2--Exception-Syndrome-Register--EL2- const auto* ctx = reinterpret_cast(ucontext); const auto esr = ctx == nullptr ? 0 : ctx->uc_mcontext->__es.__esr; constexpr decltype(esr) fpe_mask(0b10110000000000000000000000000000); if ((fpe_mask & esr) == fpe_mask) { if (0 != (esr & 0b1)) { LOG("esr: IOF"); } if (0 != (esr & 0b10)) { LOG("esr: DZF"); } if (0 != (esr & 0b0100)) { LOG("esr: OFF"); } if (0 != (esr & 0b1000)) { LOG("esr: UFF"); } if (0 != (esr & 0b10000)) { LOG("esr: IXF"); } if (0 != (esr & 0b10000000)) { LOG("esr: IDF"); } } } #endif else { LOG("signal: ?"); } LOG("---"); #undef LOG #undef STR LibEcKit::instance().abort(); // Just in case we end up here, which normally we shouldn't. std::abort(); } void enable_signal_handlers(bool force) { if (force || CUSTOM_EXCEPTS != 0) { CUSTOM_SIGNAL_HANDLERS.set(SIGFPE); #if defined(__APPLE__) && defined(__arm64__) CUSTOM_SIGNAL_HANDLERS.set(SIGILL); #endif } } void disable_signal_handlers() { CUSTOM_SIGNAL_HANDLERS.unset(); } } // namespace //---------------------------------------------------------------------------------------------------------------------- void FloatingPointExceptions::enable_floating_point_exceptions(const std::string& codes) { if (auto excepts = make_excepts(codes); excepts != 0) { enable_floating_point_exception(excepts); } } void FloatingPointExceptions::disable_floating_point_exceptions(const std::string& codes) { if (auto excepts = make_excepts(codes); excepts != 0) { disable_floating_point_exception(excepts); } } void FloatingPointExceptions::enable_custom_signal_handlers(bool force) { enable_signal_handlers(force); } void FloatingPointExceptions::disable_custom_signal_handlers() { disable_signal_handlers(); } void FloatingPointExceptions::test(const std::string& codes) { auto excepts = make_excepts(codes); if (0 != (excepts & FE_INVALID)) { // invalid operation volatile double x = 0.; volatile double y = 0.; volatile double z = x / y; (void)z; // prevent optimization } if (0 != (excepts & FE_INEXACT)) { // inexact result volatile double x = 1 / 3.; volatile double y = x * 3.; (void)y; } if (0 != (excepts & FE_DIVBYZERO)) { // divide-by-zero exception volatile double x = 1.; volatile double y = 0.; volatile double z = x / y; (void)z; } if (0 != (excepts & FE_OVERFLOW)) { // floating-point overflow volatile double x = 1.e308; volatile double y = x * x; (void)y; } if (0 != (excepts & FE_UNDERFLOW)) { // floating-point underflow volatile double x = 1.e-308; volatile double y = x * x; (void)y; } #if defined(FE_DENORMAL) if (0 != (excepts & FE_DENORMAL)) { // use of a denormalized floating-point number volatile double x = 1.e-320; (void)x; } #endif } } // namespace eckit::maths eckit-2.0.7/src/eckit/maths/Matrix3.h0000664000175000017500000001003715161702250017505 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include "eckit/exception/Exceptions.h" #include "eckit/geo/PointXYZ.h" namespace eckit::maths { template class Matrix3 final : protected std::array { public: // -- Types using container_type = std::array; public: // -- Constructors Matrix3(T xx, T xy, T xz, T yx, T yy, T yz, T zx, T zy, T zz) : container_type{xx, xy, xz, yx, yy, yz, zx, zy, zz} {} Matrix3(const Matrix3& other) : container_type(other) {} Matrix3(Matrix3&& other) : container_type(other) {} // -- Destructor ~Matrix3() = default; // -- Operators Matrix3& operator=(const Matrix3& other) { container_type::operator=(other); return *this; } Matrix3& operator=(Matrix3&& other) { container_type::operator=(other); return *this; } geo::PointXYZ operator*(const geo::PointXYZ& p) const { return {XX() * p.X() + XY() * p.Y() + XZ() * p.Z(), // YX() * p.X() + YY() * p.Y() + YZ() * p.Z(), // ZX() * p.X() + ZY() * p.Y() + ZZ() * p.Z()}; } Matrix3 operator*(const Matrix3& M) const { return {XX() * M.XX() + XY() * M.YX() + XZ() * M.ZX(), XX() * M.XY() + XY() * M.YY() + XZ() * M.ZY(), XX() * M.XZ() + XY() * M.YZ() + XZ() * M.ZZ(), YX() * M.XX() + YY() * M.YX() + YZ() * M.ZX(), YX() * M.XY() + YY() * M.YY() + YZ() * M.ZY(), YX() * M.XZ() + YY() * M.YZ() + YZ() * M.ZZ(), ZX() * M.XX() + ZY() * M.YX() + ZZ() * M.ZX(), ZX() * M.XY() + ZY() * M.YY() + ZZ() * M.ZY(), ZX() * M.XZ() + ZY() * M.YZ() + ZZ() * M.ZZ()}; } // -- Members const T& XX() const { return container_type::operator[](0); } const T& XY() const { return container_type::operator[](1); } const T& XZ() const { return container_type::operator[](2); } const T& YX() const { return container_type::operator[](3); } const T& YY() const { return container_type::operator[](4); } const T& YZ() const { return container_type::operator[](5); } const T& ZX() const { return container_type::operator[](6); } const T& ZY() const { return container_type::operator[](7); } const T& ZZ() const { return container_type::operator[](8); } // -- Methods using container_type::begin; using container_type::cbegin; using container_type::cend; using container_type::end; using container_type::size; static Matrix3 identity() { return {1, 0, 0, 0, 1, 0, 0, 0, 1}; } Matrix3 inverse() const { auto det = XX() * (YY() * ZZ() - YZ() * ZY()) - XY() * (YX() * ZZ() - YZ() * ZX()) + XZ() * (YX() * ZY() - YY() * ZX()); ASSERT_MSG(det != 0, "Matrix3: singular matrix"); return { (YY() * ZZ() - YZ() * ZY()) / det, (XZ() * ZY() - XY() * ZZ()) / det, (XY() * YZ() - XZ() * YY()) / det, (YZ() * ZX() - YX() * ZZ()) / det, (XX() * ZZ() - XZ() * ZX()) / det, (XZ() * YX() - XX() * YZ()) / det, (YX() * ZY() - YY() * ZX()) / det, (XY() * ZX() - XX() * ZY()) / det, (XX() * YY() - XY() * YX()) / det}; } T determinant() const { return XX() * YY() * ZZ() - XZ() * YY() * ZX() + XY() * YZ() * ZX() + XZ() * YX() * ZY() - XX() * YZ() * ZY() - XY() * YX() * ZZ(); } // -- Friends friend std::ostream& operator<<(std::ostream& out, const Matrix3& m) { return out << "{{" << m.XX() << ", " << m.XY() << ", " << m.XZ() << "}, {" << m.YX() << ", " << m.YY() << ", " << m.YZ() << "}, {" << m.ZX() << ", " << m.ZY() << ", " << m.ZZ() << "}}"; } }; } // namespace eckit::maths eckit-2.0.7/src/eckit/maths/Functions.cc0000664000175000017500000000142215161702250020262 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/maths/Functions.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- size_t round(size_t x, size_t n) { return ((x + n - 1) / n) * n; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/maths/MatrixLapack.h0000664000175000017500000005750215161702250020546 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /** * @file * @author Willem Deconinck * @date Sept 2014 * * Do not include this file directly. Rather include "eckit/maths/Matrix.h" instead. * * This file implements classes * - RowVector * - ColVector * - Matrix * * These classes follow Eigen's naming of functions, and could be used * as a drop-in replacement, depending on compilation choices * * The reasoning is that some compilers (e.g. CRAY) have problems compiling * Eigen. * * Where possible, BLAS and LAPACK implementations are used, in case * BLAS or LAPACK have been found. */ #ifndef eckit_maths_Matrix_h #error Include "eckit/maths/Matrix.h" instead of "eckit/maths/MatrixLapack.h" #endif #ifndef eckit_maths_MatrixLapack_h #define eckit_maths_MatrixLapack_h #include #include #include "eckit/exception/Exceptions.h" #include "eckit/maths/Lapack.h" #ifdef minor #undef minor #endif namespace eckit { namespace maths { namespace detail { namespace ColMajor_4x4 { template struct remove_const { using type = T; }; template struct remove_const { using type = T; }; template inline void invert(Scalar m[16], typename remove_const::type inv[16]); template inline Scalar det(Scalar m[16]); } // namespace ColMajor_4x4 } // namespace detail template class Matrix { protected: Scalar* data_{nullptr}; Index nr_{0}; Index nc_{0}; bool is_proxy_{false}; public: using MapType = Matrix; using ConstMapType = MapType; Matrix() = default; // Constructor that allocates matrix with sizes template Matrix(const T0& nr, const T1& nc) { resize(nr, nc); } template Matrix(Scalar* data, const T0& nr, const T1& nc) : data_{data}, nr_{nr}, nc_{nc}, is_proxy_{true} {} // This constructor allows you to construct Matrix from Eigen expressions Matrix(const Matrix& other) { resize(other.nr_, other.nc_); std::memcpy(data_, other.data(), sizeof(Scalar) * nr_ * nc_); } // Constructor that allocates matrix and infers sizes from initializer list Matrix(const std::initializer_list >& list) { Index nr = list.size(); if (nr > 0) { Index nc = list.begin()->size(); ASSERT(nc > 0); resize(nr, nc); Index i{0}; for (const auto& row : list) { ASSERT(row.size() == nc); Index j{0}; for (const auto& elem : row) { at(i, j) = elem; ++j; } ++i; } } } ~Matrix() { if (!is_proxy_) { delete[] data_; } } Matrix& noalias() { return *this; } void resize(Index nr, Index nc) { if (is_proxy_) { if (nr != nr_ || nc != nc_) { throw eckit::Exception("Illegal call: Trying to resize a proxy matrix", Here()); } } else { if (nr != nr_ || nc != nc_) { if (data_) { delete[] data_; } data_ = new Scalar[nr * nc]; nr_ = nr; nc_ = nc; } } } void setZero() { ASSERT(size() > 0); ASSERT(data_); ::memset(data_, 0, size() * sizeof(Scalar)); } Matrix& operator=(const Matrix& other) { resize(other.nr_, other.nc_); std::memcpy(data_, other.data(), sizeof(Scalar) * nr_ * nc_); return *this; } Scalar* data() { return data_; } const Scalar* data() const { return data_; } Index size() const { return nr_ * nc_; } Index rows() const { return nr_; } Index cols() const { return nc_; } template Scalar& operator[](const T& i) { return data_[i]; } template const Scalar& operator[](const T& i) const { return data_[i]; } template Scalar& operator()(const T& i) { return data_[i]; } template const Scalar& operator()(const T& i) const { return data_[i]; } template Scalar& operator()(const T0& i, const T1& j) { return at(i, j); } template const Scalar& operator()(const T0& i, const T1& j) const { return at(i, j); } template Matrix row(const T& i) const { Matrix r(1, nc_); for (Index j = 0; j < nc_; ++j) r(j) = at(i, j); return r; } template Matrix col(const T& j) const { Matrix c(nc_, 1); for (Index i = 0; i < nc_; ++i) c(i) = at(i, j); return c; } Matrix inverse() const { Matrix inv(nr_, nc_); switch (nr_) { case 1: { inv(0, 0) = 1. / at(0, 0); break; } case 2: { Scalar deti = 1. / (at(0, 0) * at(1, 1) - at(0, 1) * at(1, 0)); inv(0, 0) = at(1, 1) * deti; inv(0, 1) = -at(0, 1) * deti; inv(1, 0) = -at(1, 0) * deti; inv(1, 1) = at(0, 0) * deti; break; } case 3: { Scalar deti = 1. / (at(0, 0) * at(1, 1) * at(2, 2) + at(1, 0) * at(2, 1) * at(0, 2) + at(2, 0) * at(0, 1) * at(1, 2) - at(0, 0) * at(2, 1) * at(1, 2) - at(2, 0) * at(1, 1) * at(0, 2) - at(1, 0) * at(0, 1) * at(2, 2)); inv(0, 0) = at(1, 1) * at(2, 2) - at(1, 2) * at(2, 1); inv(0, 1) = at(0, 2) * at(2, 1) - at(0, 1) * at(2, 2); inv(0, 2) = at(0, 1) * at(1, 2) - at(0, 2) * at(1, 1); inv(1, 0) = at(1, 2) * at(2, 0) - at(1, 0) * at(2, 2); inv(1, 1) = at(0, 0) * at(2, 2) - at(0, 2) * at(2, 0); inv(1, 2) = at(0, 2) * at(1, 0) - at(0, 0) * at(1, 2); inv(2, 0) = at(1, 0) * at(2, 1) - at(1, 1) * at(2, 0); inv(2, 1) = at(0, 1) * at(2, 0) - at(0, 0) * at(2, 1); inv(2, 2) = at(0, 0) * at(1, 1) - at(0, 1) * at(1, 0); for (Index i = 0; i < 9; ++i) inv.data()[i] *= deti; break; } case 4: { detail::ColMajor_4x4::invert(data(), inv.data()); break; } default: { // invert with LU-factorization #if eckit_HAVE_LAPACK inv = *this; int M = nr_; int N = nc_; int lda = nr_; int info; int* ipiv = new int[std::min(M, N)]; lapack::getrf(&M, &N, inv.data(), &lda, ipiv, &info); if (info == 0) { int lwork = M * 4; double* work = new double[lwork]; lapack::getri(&M, inv.data(), &lda, ipiv, work, &lwork, &info); delete[] work; } delete[] ipiv; if (info > 0) { std::stringstream stream; stream << "U(" << info << "," << info << ")=0 in matrix inversion"; throw eckit::Exception(stream.str(), Here()); } else if (info < 0) { std::stringstream stream; stream << "Wrong " << -info << "-th argument in matrix inversion"; throw eckit::Exception(stream.str(), Here()); } #else // This algorithm inverts a matrix based on the Gauss Jordan method. int n = nr_; Scalar det, pivot, factor; Matrix work(n, n); det = 1; for (int i = 0; i < n; i++) { for (int j = 0; j < n; j++) { inv(i, j) = 0; work(i, j) = at(i, j); } inv(i, i) = 1.; } // The current pivot row is jpass. // For each pass, first find the maximum element in the pivot column. for (int jpass = 0; jpass < n; jpass++) { int imx = jpass; for (int jrow = jpass; jrow < n; jrow++) { if (std::abs(work(jrow, jpass)) > std::abs(work(imx, jpass))) imx = jrow; } // Interchange the elements of row jpass and row imx in both A and AInverse. if (imx != jpass) { for (int jcol = 0; jcol < n; jcol++) { Scalar temp = inv(jpass, jcol); inv(jpass, jcol) = inv(imx, jcol); inv(imx, jcol) = temp; if (jcol >= jpass) { temp = work(jpass, jcol); work(jpass, jcol) = work(imx, jcol); work(imx, jcol) = temp; } } } // The current pivot is now A[jpass][jpass]. // The determinant is the product of the pivot elements. pivot = work(jpass, jpass); det *= pivot; if (det == 0) { throw eckit::SeriousBug("Cannot invert zero determinant matrix", Here()); } for (int jcol = 0; jcol < n; jcol++) { // Normalize the pivot row by dividing by the pivot element. inv(jpass, jcol) = inv(jpass, jcol) / pivot; if (jcol >= jpass) work(jpass, jcol) = work(jpass, jcol) / pivot; } for (int jrow = 0; jrow < n; jrow++) // Add a multiple of the pivot row to each row. The multiple factor // is chosen so that the element of A on the pivot column is 0. { if (jrow != jpass) factor = work(jrow, jpass); for (int jcol = 0; jcol < n; jcol++) { if (jrow != jpass) { inv(jrow, jcol) -= factor * inv(jpass, jcol); work(jrow, jcol) -= factor * work(jpass, jcol); } } } } #endif } } return inv; } Matrix transpose() const { Matrix transposed(nc_, nr_); for (Index i = 0; i < nr_; ++i) for (Index j = 0; j < nc_; ++j) transposed(j, i) = at(i, j); return transposed; } Scalar determinant() const { switch (nr_) { case 1: { return at(0, 0); } case 2: { return (at(0, 0) * at(1, 1) - at(0, 1) * at(1, 0)); } case 3: { return (at(0, 0) * at(1, 1) * at(2, 2) + at(1, 0) * at(2, 1) * at(0, 2) + at(2, 0) * at(0, 1) * at(1, 2) - at(0, 0) * at(2, 1) * at(1, 2) - at(2, 0) * at(1, 1) * at(0, 2) - at(1, 0) * at(0, 1) * at(2, 2)); } case 4: { return detail::ColMajor_4x4::det(data()); } default: { Scalar d = 0; // value of the determinant // this is a matrix of 5x5 or larger for (int c = 0; c < cols(); c++) { Matrix M = minor(0, c); d += (2 * ((c + 1) % 2) - 1) * at(0, c) * M.determinant(); } return d; } } } template friend std::ostream& operator<<(std::ostream& os, const Matrix& v); #define UNARY_OPERATOR_Scalar(OP) \ Matrix& operator OP(const Scalar & scal) { \ for (Index i = 0; i < size(); ++i) \ data_[i] OP scal; \ return *this; \ } UNARY_OPERATOR_Scalar(+=) UNARY_OPERATOR_Scalar(-=) UNARY_OPERATOR_Scalar(*=) UNARY_OPERATOR_Scalar(/=) #undef UNARY_OPERATOR_Scalar // Matrix Matrix multiply Matrix dot(const Matrix& m) const { // Very naive implementation. We should use BLAS later Matrix result(rows(), m.cols()); for (Index i = 0; i < rows(); ++i) { for (Index j = 0; j < m.cols(); ++j) { result(i, j) = 0.; for (Index k = 0; k < cols(); ++k) result(i, j) += at(i, k) * m(k, j); } } return result; } Matrix cwiseProduct(const Matrix& m) const { Matrix result(*this); for (Index i = 0; i < size(); ++i) result.data_[i] *= m.data_[i]; return result; } Matrix operator+(const Matrix& m) const { Matrix result(*this); result += m; return result; } Matrix operator-(const Matrix& m) const { Matrix result(*this); result -= m; return result; } // Matrix Matrix multiply Matrix operator*(const Matrix& m) const { return dot(m); } Matrix& operator+=(const Matrix& m) { for (Index i = 0; i < size(); ++i) data_[i] += m.data_[i]; return *this; } Matrix& operator-=(const Matrix& m) { for (Index i = 0; i < size(); ++i) data_[i] -= m.data_[i]; return *this; } bool operator==(const Matrix& m) const { if (rows() != m.rows() || cols() != m.cols()) { return false; } for (Index i = 0; i < size(); ++i) { if (data_[i] != m.data_[i]) { return false; } } return true; } // Matrix Scalar multiply Matrix operator*(const Scalar& s) const { Matrix m(*this); m *= s; return m; } // Matrix Scalar divide Matrix operator/(const Scalar& s) const { ASSERT(s != 0.0); Matrix m(*this); m /= s; return m; } private: Scalar& at(const Index i, const Index j) { return data_[i + nr_ * j]; } const Scalar& at(const Index i, const Index j) const { if (!data_) throw eckit::Exception("data_ is null", Here()); return data_[i + nr_ * j]; } template Matrix minor(const T0& row, const T1& col) const { Matrix m(rows() - 1, cols() - 1); Index mr(0); for (Index r = 0; r < rows(); ++mr) { if (r == row) ++r; if (r == rows()) break; Index mc(0); for (Index c = 0; c < (cols()); ++mc) { if (c == col) ++c; if (c == cols()) break; m(mr, mc) = at(r, c); ++c; } ++r; } return m; } }; template class ColVector; template class RowVector : public Matrix { using Base = Matrix; public: using MapType = RowVector; using ConstMapType = MapType; public: RowVector() : Base() {} // Constructor that allocates matrix with sizes template RowVector(const T0& nc) : Base(1, nc) {} template RowVector(Scalar* data, const T0& nc) : Base(data, 1, nc) {} // This constructor allows you to construct Matrix from Eigen expressions RowVector(const Base& other) : Base(other) {} void resize(Index nc) { Base::resize(1, nc); } RowVector& operator=(const Base& other) { resize(other.cols()); std::memcpy(this->data(), other.data(), sizeof(Scalar) * this->cols()); return *this; } // RowVector ColVector multiply Scalar operator*(const ColVector& c) const { Scalar s(0); for (Index j = 0; j < Base::nc_; ++j) s += Base::data_[j] * c[j]; return s; } // RowVector Matrix multiply Base operator*(const Base& m) const { return Base::operator*(m); } }; template class ColVector : public Matrix { using Base = Matrix; public: using MapType = ColVector; using ConstMapType = MapType; public: ColVector() : Base() {} // Constructor that allocates matrix with sizes template ColVector(const T0& nr) : Base(nr, 1) {} template ColVector(Scalar* data, const T0& nr) : Base(data, nr, 1) {} // This constructor allows you to construct Matrix from Eigen expressions ColVector(const Base& other) : Base(other) {} void resize(Index nr) { Base::resize(nr, 1); } ColVector& operator=(const Base& other) { resize(other.rows()); std::memcpy(this->data(), other.data(), sizeof(Scalar) * this->rows()); return *this; } // ColVector RowVector multiply Matrix operator*(const RowVector& r) const { Matrix m(Base::nc_, r.nr_); for (Index i = 0; i < m.nr_; ++i) { for (Index j = 0; j < m.nc_; ++j) { m(i, j) = Base::data_[i] * r[j]; } } return m; } }; template std::ostream& operator<<(std::ostream& os, const Matrix& m) { for (Index i = 0; i < m.nr_; ++i) { if (i > 0) os << "\n"; for (Index j = 0; j < m.nc_; ++j) { os << m(i, j) << " "; } } return os; } namespace detail { namespace ColMajor_4x4 { template inline void invert(Scalar m[16], typename remove_const::type inv[16]) { inv[0] = m[5] * m[10] * m[15] - m[5] * m[11] * m[14] - m[9] * m[6] * m[15] + m[9] * m[7] * m[14] + m[13] * m[6] * m[11] - m[13] * m[7] * m[10]; inv[4] = -m[4] * m[10] * m[15] + m[4] * m[11] * m[14] + m[8] * m[6] * m[15] - m[8] * m[7] * m[14] - m[12] * m[6] * m[11] + m[12] * m[7] * m[10]; inv[8] = m[4] * m[9] * m[15] - m[4] * m[11] * m[13] - m[8] * m[5] * m[15] + m[8] * m[7] * m[13] + m[12] * m[5] * m[11] - m[12] * m[7] * m[9]; inv[12] = -m[4] * m[9] * m[14] + m[4] * m[10] * m[13] + m[8] * m[5] * m[14] - m[8] * m[6] * m[13] - m[12] * m[5] * m[10] + m[12] * m[6] * m[9]; inv[1] = -m[1] * m[10] * m[15] + m[1] * m[11] * m[14] + m[9] * m[2] * m[15] - m[9] * m[3] * m[14] - m[13] * m[2] * m[11] + m[13] * m[3] * m[10]; inv[5] = m[0] * m[10] * m[15] - m[0] * m[11] * m[14] - m[8] * m[2] * m[15] + m[8] * m[3] * m[14] + m[12] * m[2] * m[11] - m[12] * m[3] * m[10]; inv[9] = -m[0] * m[9] * m[15] + m[0] * m[11] * m[13] + m[8] * m[1] * m[15] - m[8] * m[3] * m[13] - m[12] * m[1] * m[11] + m[12] * m[3] * m[9]; inv[13] = m[0] * m[9] * m[14] - m[0] * m[10] * m[13] - m[8] * m[1] * m[14] + m[8] * m[2] * m[13] + m[12] * m[1] * m[10] - m[12] * m[2] * m[9]; inv[2] = m[1] * m[6] * m[15] - m[1] * m[7] * m[14] - m[5] * m[2] * m[15] + m[5] * m[3] * m[14] + m[13] * m[2] * m[7] - m[13] * m[3] * m[6]; inv[6] = -m[0] * m[6] * m[15] + m[0] * m[7] * m[14] + m[4] * m[2] * m[15] - m[4] * m[3] * m[14] - m[12] * m[2] * m[7] + m[12] * m[3] * m[6]; inv[10] = m[0] * m[5] * m[15] - m[0] * m[7] * m[13] - m[4] * m[1] * m[15] + m[4] * m[3] * m[13] + m[12] * m[1] * m[7] - m[12] * m[3] * m[5]; inv[14] = -m[0] * m[5] * m[14] + m[0] * m[6] * m[13] + m[4] * m[1] * m[14] - m[4] * m[2] * m[13] - m[12] * m[1] * m[6] + m[12] * m[2] * m[5]; inv[3] = -m[1] * m[6] * m[11] + m[1] * m[7] * m[10] + m[5] * m[2] * m[11] - m[5] * m[3] * m[10] - m[9] * m[2] * m[7] + m[9] * m[3] * m[6]; inv[7] = m[0] * m[6] * m[11] - m[0] * m[7] * m[10] - m[4] * m[2] * m[11] + m[4] * m[3] * m[10] + m[8] * m[2] * m[7] - m[8] * m[3] * m[6]; inv[11] = -m[0] * m[5] * m[11] + m[0] * m[7] * m[9] + m[4] * m[1] * m[11] - m[4] * m[3] * m[9] - m[8] * m[1] * m[7] + m[8] * m[3] * m[5]; inv[15] = m[0] * m[5] * m[10] - m[0] * m[6] * m[9] - m[4] * m[1] * m[10] + m[4] * m[2] * m[9] + m[8] * m[1] * m[6] - m[8] * m[2] * m[5]; typename remove_const::type det = m[0] * inv[0] + m[1] * inv[4] + m[2] * inv[8] + m[3] * inv[12]; if (det == 0) { Matrix mat(m, 4, 4); std::stringstream msg; msg << "Trying to invert 4x4 matrix with zero determinant.\nMatrix = \n" << mat; throw eckit::Exception(msg.str(), Here()); } det = 1. / det; for (int i = 0; i < 16; i++) inv[i] *= det; } template Scalar det(Scalar m[16]) { Scalar inv[4] = { m[5] * m[10] * m[15] - m[5] * m[11] * m[14] - m[9] * m[6] * m[15] + m[9] * m[7] * m[14] + m[13] * m[6] * m[11] - m[13] * m[7] * m[10], -m[4] * m[10] * m[15] + m[4] * m[11] * m[14] + m[8] * m[6] * m[15] - m[8] * m[7] * m[14] - m[12] * m[6] * m[11] + m[12] * m[7] * m[10], m[4] * m[9] * m[15] - m[4] * m[11] * m[13] - m[8] * m[5] * m[15] + m[8] * m[7] * m[13] + m[12] * m[5] * m[11] - m[12] * m[7] * m[9], -m[4] * m[9] * m[14] + m[4] * m[10] * m[13] + m[8] * m[5] * m[14] - m[8] * m[6] * m[13] - m[12] * m[5] * m[10] + m[12] * m[6] * m[9], }; return m[0] * inv[0] + m[1] * inv[1] + m[2] * inv[2] + m[3] * inv[3]; } } // namespace ColMajor_4x4 namespace RowMajor_4x4 { // minor template inline Scalar mnr(Scalar m[16], int r0, int r1, int r2, int c0, int c1, int c2) { return m[4 * r0 + c0] * (m[4 * r1 + c1] * m[4 * r2 + c2] - m[4 * r2 + c1] * m[4 * r1 + c2]) - m[4 * r0 + c1] * (m[4 * r1 + c0] * m[4 * r2 + c2] - m[4 * r2 + c0] * m[4 * r1 + c2]) + m[4 * r0 + c2] * (m[4 * r1 + c0] * m[4 * r2 + c1] - m[4 * r2 + c0] * m[4 * r1 + c1]); } template inline void adjoint(Scalar m[16], Scalar adj[16]) { adj[0] = mnr(m, 1, 2, 3, 1, 2, 3); adj[1] = -mnr(m, 0, 2, 3, 1, 2, 3); adj[2] = mnr(m, 0, 1, 3, 1, 2, 3); adj[3] = -mnr(m, 0, 1, 2, 1, 2, 3); adj[4] = -mnr(m, 1, 2, 3, 0, 2, 3); adj[5] = mnr(m, 0, 2, 3, 0, 2, 3); adj[6] = -mnr(m, 0, 1, 3, 0, 2, 3); adj[7] = mnr(m, 0, 1, 2, 0, 2, 3); adj[8] = mnr(m, 1, 2, 3, 0, 1, 3); adj[9] = -mnr(m, 0, 2, 3, 0, 1, 3); adj[10] = mnr(m, 0, 1, 3, 0, 1, 3); adj[11] = -mnr(m, 0, 1, 2, 0, 1, 3); adj[12] = -mnr(m, 1, 2, 3, 0, 1, 2); adj[13] = mnr(m, 0, 2, 3, 0, 1, 2); adj[14] = -mnr(m, 0, 1, 3, 0, 1, 2); adj[15] = mnr(m, 0, 1, 2, 0, 1, 2); } template inline Scalar det(Scalar m[16]) { return m[0] * mnr(m, 1, 2, 3, 1, 2, 3) - m[1] * mnr(m, 1, 2, 3, 0, 2, 3) + m[2] * mnr(m, 1, 2, 3, 0, 1, 3) - m[3] * mnr(m, 1, 2, 3, 0, 1, 2); } template inline void invert(Scalar m[16], Scalar inv[16]) { adjoint(m, inv); Scalar inv_det = 1.0 / det(m); for (int i = 0; i < 16; ++i) inv[i] = inv[i] * inv_det; } } // namespace RowMajor_4x4 } // namespace detail } // namespace maths } // namespace eckit #endif eckit-2.0.7/src/eckit/maths/Lapack.h0000664000175000017500000000075715161702250017361 0ustar alastairalastair#ifndef eckit_maths_lapack_h #define eckit_maths_lapack_h #include "eckit/eckit.h" namespace eckit::maths::lapack { void getrf(int* M, int* N, float* data, int* lda, int* ipiv, int* info); void getrf(int* M, int* N, double* data, int* lda, int* ipiv, int* info); void getri(int* M, float* data, int* lda, int* ipiv, float* work, int* lwork, int* info); void getri(int* M, double* data, int* lda, int* ipiv, double* work, int* lwork, int* info); } // namespace eckit::maths::lapack #endif eckit-2.0.7/src/eckit/maths/Qhull.cc0000664000175000017500000001116015161702250017377 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/maths/Qhull.h" #include #include #include "eckit/log/Log.h" #include "eckit/maths/ConvexHull.h" #include "libqhullcpp/Qhull.h" #include "libqhullcpp/QhullFacetList.h" #include "libqhullcpp/QhullVertexSet.h" namespace eckit::maths { const char* Qhull::COMMAND_DEFAULT = "QJ"; Qhull::Qhull(size_t N, const coord_t& coord, const std::string& command) { ASSERT(0 < N && coord.size() % N == 0); auto pointDimension = static_cast(N); auto pointCount = static_cast(coord.size() / N); qh_ = new orgQhull::Qhull; ASSERT(qh_ != nullptr); std::ostringstream err; qh_->setErrorStream(&err); qh_->setOutputStream(&Log::info()); qh_->enableOutputStream(); try { qh_->runQhull("", pointDimension, pointCount, coord.data(), command.c_str()); ASSERT(qh_->qhullStatus() == 0); } catch (const orgQhull::QhullError& e) { static const std::set DIMENSION_ERROR{6050}; static const std::set INPUT_ERROR{6010, 6013, 6019, 6020, 6021, 6022, 6023, 6033, 6067, 6070, 6072, 6073, 6074, 6075, 6077, 6078, 6150, 6151, 6152, 6153, 6203, 6204, 6214, 6220, 6221, 6222, 6223, 6229, 6411, 6412}; static const std::set OPTION_ERROR{6006, 6029, 6035, 6036, 6037, 6041, 6044, 6045, 6046, 6047, 6048, 6049, 6051, 6053, 6054, 6055, 6056, 6057, 6058, 6059, 6215, 6362, 6363, 6364, 6365, 6375}; static const std::set PRECISION_ERROR{6012, 6109, 6110, 6111, 6112, 6113, 6114, 6115, 6116, 6117, 6118, 6136, 6154, 6239, 6240, 6297, 6298, 6347, 6348, 6354, 6379, 6380, 6417, 6418, 6422}; static const std::set TOPOLOGY_ERROR{6001, 6107, 6155, 6168, 6170, 6208, 6227, 6260, 6271, 6356, 6361, 6381, 6391, 6420, 6425}; auto is = [](int err, const auto& set) { return set.find(err) != set.end(); }; is(e.errorCode(), DIMENSION_ERROR) ? throw ConvexHull::DimensionError(err.str(), e.errorCode(), command) : is(e.errorCode(), INPUT_ERROR) ? throw ConvexHull::InputError(err.str(), e.errorCode(), command) : is(e.errorCode(), OPTION_ERROR) ? throw ConvexHull::OptionError(err.str(), e.errorCode(), command) : is(e.errorCode(), PRECISION_ERROR) ? throw ConvexHull::PrecisionError(err.str(), e.errorCode(), command) : is(e.errorCode(), TOPOLOGY_ERROR) ? throw ConvexHull::TopologyError(err.str(), e.errorCode(), command) : throw ConvexHull::Exception(err.str(), e.errorCode(), command); } } Qhull::~Qhull() { delete qh_; } std::vector Qhull::list_vertices() const { std::vector vertices; vertices.reserve(qh_->vertexCount()); for (const auto& vertex : qh_->vertexList()) { vertices.emplace_back(vertex.point().id()); } return vertices; } std::vector> Qhull::list_facets() const { std::vector> facets; facets.reserve(qh_->facetCount()); for (const auto& facet : qh_->facetList()) { const auto vertices = facet.vertices(); std::vector f; f.reserve(vertices.size()); for (const auto& vertex : vertices) { f.emplace_back(vertex.point().id()); } facets.emplace_back(std::move(f)); } return facets; } Qhull::facets_n_t Qhull::facets_n() const { facets_n_t counts; for (const auto& facet : qh_->facetList()) { counts[facet.vertices().size()]++; } return counts; } std::vector Qhull::facets(size_t n) const { ASSERT(0 < n); std::vector facets; facets.reserve(qh_->facetCount() * n); for (const auto& facet : qh_->facetList()) { if (const auto vertices = facet.vertices(); vertices.size() == n) { for (const auto& vertex : vertices) { facets.emplace_back(vertex.point().id()); } } } return facets; } } // namespace eckit::maths eckit-2.0.7/src/eckit/maths/Matrix.h0000664000175000017500000000262215161702250017423 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /** * @file * @author Willem Deconinck * @date Sept 2014 * * This file introduces classes * - RowVector * - ColVector * - Matrix * * By default, they are straight inherited from Eigen classes * Due to current inability to compile Eigen on CRAY, * Transitional classes have been created, which only implement * a small subset of Eigen functionality, and to let at least * code be compiled * It is strongly advised to only use the subset in the * transitional classes, or implement as needed */ #ifndef eckit_maths_Matrix_h #define eckit_maths_Matrix_h #include "eckit/eckit.h" namespace eckit::maths { template class RowVector; template class ColVector; template class Matrix; } // namespace eckit::maths #if eckit_HAVE_EIGEN // Implementation using Eigen #include "eckit/maths/MatrixEigen.h" #else // Own implementation using Lapack if available #include "eckit/maths/MatrixLapack.h" #endif #endif eckit-2.0.7/src/eckit/maths/Functions.h0000664000175000017500000000207515161702250020131 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date May 2016 #ifndef eckit_Functions_h #define eckit_Functions_h #include #include namespace eckit { //---------------------------------------------------------------------------------------------------------------------- /// @returns rounds x to multiple of n size_t round(size_t x, size_t n); /// @returns the sign of ordereable types that support the subtraction operator template int sign(T v) { return (T(0) < v) - (v < T(0)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/maths/ConvexHullN.h0000664000175000017500000000435015161702250020364 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/maths/ConvexHull.h" namespace eckit::maths { template class ConvexHullN : public ConvexHull { public: explicit ConvexHullN(const ConvexHull::coord_t& coord, const std::string& qhull_command = Qhull::COMMAND_DEFAULT) : qhull_(N, coord, qhull_command) {} explicit ConvexHullN(const std::vector>& coord_v, const std::string& qhull_command = Qhull::COMMAND_DEFAULT) : ConvexHullN(convert_vector_v(coord_v), qhull_command) {} explicit ConvexHullN(const std::vector>& coord_a, const std::string& qhull_command = Qhull::COMMAND_DEFAULT) : ConvexHullN(convert_vector_a(coord_a), qhull_command) {} std::vector list_vertices() const override { return qhull_.list_vertices(); } std::vector> list_facets() const override { return qhull_.list_facets(); } ConvexHull::facets_n_t facets_n() const override { return qhull_.facets_n(); } std::vector facets(size_t n) const override { return qhull_.facets(n); } private: static coord_t convert_vector_v(const std::vector>& coord_v) { coord_t coord; coord.reserve(N * coord_v.size()); for (const auto& v : coord_v) { ASSERT(N == v.size()); std::for_each_n(v.begin(), N, [&coord](auto x) { coord.emplace_back(x); }); } return coord; } static coord_t convert_vector_a(const std::vector>& coord_a) { coord_t coord; coord.reserve(N * coord_a.size()); for (const auto& a : coord_a) { std::for_each_n(a.begin(), N, [&coord](auto x) { coord.emplace_back(x); }); } return coord; } Qhull qhull_; }; } // namespace eckit::maths eckit-2.0.7/src/eckit/maths/MatrixEigen.h0000664000175000017500000001144415161702250020375 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_maths_Matrix_h #error Include "eckit/maths/Matrix.h" instead of "eckit/maths/MatrixEigen.h" #endif #ifndef eckit_maths_MatrixEigen_h #define eckit_maths_MatrixEigen_h #include "eckit/maths/Eigen.h" #include "eckit/exception/Exceptions.h" namespace eckit { namespace maths { //======================================================================================== /// @brief Matrix class, with internal Column-Major storage ordering /// /// This class is inherited from Eigen's matrix class /// Eigen::ColMajor is also Eigen's default (recommended) storage ordering. /// Furthermore it is directly compatible with Fortran matrix manipulation routines template class Matrix : public Eigen::Matrix { using Base = Eigen::Matrix; public: // Default constructor Matrix(void) : Base() {} // Constructor that allocates matrix with sizes template Matrix(const T0& x, const T1& y) : Base(x, y) {} // Constructor that allocates matrix and infers sizes from initializer list Matrix(const std::initializer_list>& list) : #if EIGEN_VERSION_AT_LEAST(3, 4, 0) Base(list) { } #else Base() { // Old Eigen versions don't have initializer list initialization :( size_t nr = list.size(); if (nr > 0) { index nc = list.begin()->size(); ASSERT(nc > 0); this->resize(nr, nc); size_t i{0}; for (const auto& row : list) { ASSERT(row.size() == nc); size_t j{0}; for (const auto& elem : row) { (*this)(i, j) = elem; ++j; } ++i; } } } #endif // This constructor allows you to construct Matrix from Eigen expressions template Matrix(const Eigen::MatrixBase& other) : Base(other) {} // This method allows you to assign Eigen expressions to Matrix template Matrix& operator=(const Eigen::MatrixBase& other) { this->Base::operator=(other); return *this; } }; //======================================================================================== /// @brief Row-Vector class /// /// This class is inherited from Eigen's matrix class template class RowVector : public Eigen::Matrix { using Base = Eigen::Matrix; public: // Default constructor RowVector(void) : Base() {} // Constructor that allocates matrix with sizes template RowVector(const T& x) : Base(x) {} // This constructor allows you to construct Matrix from Eigen expressions template RowVector(const Eigen::MatrixBase& other) : Base(other) {} // This method allows you to assign Eigen expressions to Matrix template RowVector& operator=(const Eigen::MatrixBase& other) { this->Base::operator=(other); return *this; } }; //======================================================================================== /// @brief Column-Vector class /// /// This class is inherited from Eigen's matrix class template class ColVector : public Eigen::Matrix { using Base = Eigen::Matrix; public: // Default constructor ColVector(void) : Base() {} // Constructor that allocates matrix with sizes template ColVector(const T& x) : Base(x) {} // This constructor allows you to construct Matrix from Eigen expressions template ColVector(const Eigen::MatrixBase& other) : Base(other) {} // This method allows you to assign Eigen expressions to Matrix template ColVector& operator=(const Eigen::MatrixBase& other) { this->Base::operator=(other); return *this; } }; //======================================================================================== } // namespace maths } // namespace eckit #endif eckit-2.0.7/src/eckit/maths/Qhull.h0000664000175000017500000000246215161702250017246 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include #include namespace orgQhull { class Qhull; } namespace eckit::maths { class Qhull { public: // -- Types using coord_t = std::vector; using facets_n_t = std::map; static const char* COMMAND_DEFAULT; // -- Constructors Qhull(size_t N, const coord_t& coord, const std::string& command = COMMAND_DEFAULT); Qhull(const Qhull&) = delete; Qhull(Qhull&&) = delete; // -- Destructor ~Qhull(); // -- Operators Qhull& operator=(const Qhull&) = delete; Qhull& operator=(Qhull&&) = delete; // -- Methods std::vector list_vertices() const; std::vector> list_facets() const; facets_n_t facets_n() const; std::vector facets(size_t n) const; private: // -- Members orgQhull::Qhull* qh_; }; } // namespace eckit::maths eckit-2.0.7/src/eckit/maths/Eigen.h0000664000175000017500000000214615161702250017207 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file Eigen.h /// @author Tiago Quintino /// @date March 2014 #ifndef eckit_maths_Eigen_h #define eckit_maths_Eigen_h #include "eckit/eckit.h" //-------------------------------------------------------------------------------------------- #if eckit_HAVE_EIGEN #define EIGEN_NO_AUTOMATIC_RESIZING #define EIGEN_DONT_ALIGN #define EIGEN_DONT_VECTORIZE #ifdef ECKIT_CONTRIB_EIGEN #include "eigen3/Eigen/Core" #include "eigen3/Eigen/Dense" #include "eigen3/Eigen/Geometry" #include "eigen3/Eigen/Sparse" #else #include #include #include #include #endif #endif //-------------------------------------------------------------------------------------------- #endif eckit-2.0.7/src/eckit/maths/ConvexHull.h0000664000175000017500000000400415161702250020242 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include #include "eckit/exception/Exceptions.h" #include "eckit/maths/Qhull.h" namespace eckit::maths { class ConvexHull { public: // -- Types using coord_t = typename Qhull::coord_t; using facets_n_t = typename Qhull::facets_n_t; struct Exception : eckit::Exception { Exception(const std::string& what, int _errorCode, const std::string& _command) : eckit::Exception(what), errorCode(_errorCode), command(_command) {} const int errorCode; const std::string& command; }; struct DimensionError : Exception { using Exception::Exception; }; struct InputError : Exception { using Exception::Exception; }; struct OptionError : Exception { using Exception::Exception; }; struct PrecisionError : Exception { using Exception::Exception; }; struct TopologyError : Exception { using Exception::Exception; }; // -- Constructors ConvexHull(const ConvexHull&) = delete; ConvexHull(ConvexHull&&) = delete; // -- Destructor virtual ~ConvexHull() = default; // -- Operators void operator=(const ConvexHull&) = delete; void operator=(ConvexHull&&) = delete; // -- Methods virtual std::vector list_vertices() const = 0; virtual std::vector> list_facets() const = 0; virtual facets_n_t facets_n() const = 0; virtual std::vector facets(size_t n) const = 0; protected: // -- Constructors ConvexHull() /*noexcept*/ = default; }; } // namespace eckit::maths eckit-2.0.7/src/eckit/maths/Lapack.cc0000664000175000017500000000270415161702250017511 0ustar alastairalastair#include "eckit/maths/Lapack.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" namespace eckit::maths::lapack { #if eckit_HAVE_LAPACK extern "C" { void dgetrf_(int* M, int* N, double* A, int* lda, int* ipiv, int* info); void sgetrf_(int* M, int* N, float* A, int* lda, int* ipiv, int* info); void dgetri_(int* M, double* A, int* lda, int* ipiv, double* work, int* lwork, int* info); void sgetri_(int* M, float* A, int* lda, int* ipiv, float* work, int* lwork, int* info); } void getrf(int* M, int* N, double* data, int* lda, int* ipiv, int* info) { dgetrf_(M, N, data, lda, ipiv, info); } void getrf(int* M, int* N, float* data, int* lda, int* ipiv, int* info) { sgetrf_(M, N, data, lda, ipiv, info); } void getri(int* M, double* data, int* lda, int* ipiv, double* work, int* lwork, int* info) { dgetri_(M, data, lda, ipiv, work, lwork, info); } void getri(int* M, float* data, int* lda, int* ipiv, float* work, int* lwork, int* info) { sgetri_(M, data, lda, ipiv, work, lwork, info); } #else void getrf(int* M, int* N, double* data, int* lda, int* ipiv, int* info) { NOTIMP; } void getrf(int* M, int* N, float* data, int* lda, int* ipiv, int* info) { NOTIMP; } void getri(int* M, double* data, int* lda, int* ipiv, double* work, int* lwork, int* info) { NOTIMP; } void getri(int* M, float* data, int* lda, int* ipiv, float* work, int* lwork, int* info) { NOTIMP; } #endif } // namespace eckit::maths::lapack eckit-2.0.7/src/eckit/maths/CMakeLists.txt0000664000175000017500000000311115161702250020540 0ustar alastairalastairlist(APPEND eckit_maths_private_libs "${LAPACK_LIBRARIES}" "${BLAS_LIBRARIES}") list(APPEND eckit_maths_sources Eigen.h FloatingPointExceptions.cc FloatingPointExceptions.h Lapack.cc Lapack.h Matrix.h Matrix3.h MatrixEigen.h MatrixLapack.h ) if(CMAKE_CXX_COMPILER_ID MATCHES Intel) set_property( SOURCE FloatingPointExceptions.cc APPEND PROPERTY COMPILE_OPTIONS -fp-model precise) # See https://github.com/ecmwf/eckit/pull/206 # Without this, we get following error in a Intel Release build (-O3): # eckit/maths/FloatingPointExceptions.cc:26:14: error: '#pragma STDC FENV_ACCESS ON' is illegal when precise is disabled # | #pragma STDC FENV_ACCESS ON # | ^ endif() if(eckit_HAVE_CONVEX_HULL) list(APPEND eckit_maths_sources ConvexHull.h ConvexHullN.h Qhull.cc Qhull.h) list(APPEND eckit_maths_private_libs Qhull::Qhull) endif() ecbuild_add_library( TARGET eckit_maths TYPE SHARED INSTALL_HEADERS ALL HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/maths SOURCES ${eckit_maths_sources} PRIVATE_LIBS ${eckit_maths_private_libs} PUBLIC_LIBS eckit ) if(eckit_HAVE_EIGEN) if(NOT EIGEN3_INCLUDE_DIRS AND TARGET Eigen3::Eigen) get_target_property(EIGEN3_INCLUDE_DIRS Eigen3::Eigen INTERFACE_INCLUDE_DIRECTORIES) endif() # Add include directories with "SYSTEM" to avoid warnings from within Eigen headers target_include_directories(eckit_maths SYSTEM PUBLIC ${EIGEN3_INCLUDE_DIRS}) endif() eckit-2.0.7/src/eckit/maths/FloatingPointExceptions.h0000664000175000017500000000536015161702250023000 0ustar alastairalastair/* * (C) Copyright 2013- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include namespace eckit::maths { /** * @class FloatingPointExceptions * @brief Manage floating-point exceptions via signal handlers at runtime. * * This class allows you to enable or disable specific floating-point exceptions such as: * - FE_DIVBYZERO * - FE_INVALID * - FE_OVERFLOW * - FE_UNDERFLOW * - FE_INEXACT * * When enabled, triggering one of these exceptions will raise a SIGFPE (or SIGILL on Apple ARM64) on events such as * division by zero or overflow, and a custom signal handler will print diagnostic information and terminate the process * allowing detection and debugging of numerical issues. * * @note This is not intended for production use unless carefully managed. Enabling must happen before spawning threads, * and/or must be synchronized with any other code manipulating signal handlers. */ class FloatingPointExceptions { public: /** * @brief Enable/unmask specific floating-point exceptions. Use "FE_ALL_EXCEPT" to enable all exceptions. * @param names Comma-separated string of exception names. */ static void enable_floating_point_exceptions(const std::string& names = "FE_DIVBYZERO, FE_INVALID, FE_OVERFLOW"); /** * @brief Disable/mask specific floating-point exceptions. Use "FE_ALL_EXCEPT" to disable all exceptions. * @param names Cmma-separated string of exception names. */ static void disable_floating_point_exceptions(const std::string& names = "FE_ALL_EXCEPT"); /** * @brief Enable/install custom signal handlers to trigger on a raised floating-point exception. * * Installs custom signal handlers, platform-dependent SIGFPE (and SIGILL on Apple ARM64), saving the current * (original) ones. * * @param force Installs signal handlers if force=true, or if force=false and floating-point exceptions have been * enabled by this mechanism. */ static void enable_custom_signal_handlers(bool force = false); /** * @brief Disable/restore original custom signal handlers saved by enable_custom_signal_handlers. */ static void disable_custom_signal_handlers(); /** * @brief Test floating-point exceptions raising for the specified names. * @param names Cmma-separated string of exception names. */ static void test(const std::string& names = "FE_ALL_EXCEPT"); }; } // namespace eckit::maths eckit-2.0.7/src/eckit/eckit.h0000664000175000017500000000206615161702250016144 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Tiago Quintino /// /// @note Include this header for accessing build time configurations detected by the build system /// introspection. #ifndef eckit_eckit_h #define eckit_eckit_h #include "eckit/eckit_config.h" // configuration header auto-generated by build system //-------------------------------------------------------------------------------------------------- #if (!eckit_HAVE_MAP_ANONYMOUS) && eckit_HAVE_MAP_ANON #define MAP_ANONYMOUS MAP_ANON #endif /* Usefull macros */ #ifndef NUMBER #define NUMBER(x) (sizeof(x) / sizeof(x[0])) #endif //-------------------------------------------------------------------------------------------------- #endif eckit-2.0.7/src/eckit/exception/0000775000175000017500000000000015161702250016666 5ustar alastairalastaireckit-2.0.7/src/eckit/exception/Exceptions.h0000664000175000017500000002540415161702250021165 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /** * About adding new exceptions: * Please declare only the most general types of exceptions in this file. * This means exceptions that have a generic meaning and can be used widely. * Ideally there should already be multiple (intended) users of this new * Exception type. * * In general try to keep your domain specific exceptions close to your * implementation if there is no generic use case. See for example * eckit::geo::exception::AreaError. */ #ifndef eckit_Exceptions_h #define eckit_Exceptions_h #include #include #include #include "eckit/log/CodeLocation.h" #include "eckit/log/Log.h" #include "eckit/log/SavedStatus.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- void handle_panic(const char*, const CodeLocation& = {}); void handle_panic_no_log(const char*, const CodeLocation& = {}); void handle_assert(const std::string&, const CodeLocation& = {}); /// @brief Base class for all ECMWF exceptions. /// All exception types shall inherit from this class, either directly or /// indirectly. /// /// When inheriting from this class prefer your Exception type to be /// constructible from a message and a code location, e.g.: /// ``` /// class MyException : public Exception { /// public: /// MyException(const std::string& msg, const CodeLocation& loc = {}) : /// Exception(msg, loc){} /// } /// ``` class Exception : public std::exception { public: /// Constructors explicit Exception(const std::string& what, const CodeLocation& = {}); explicit Exception(const std::string& what, const CodeLocation& loc, bool quiet); Exception(); Exception(const Exception&) = default; Exception(Exception&&) = default; /// Destructor /// @throws nothing ~Exception() noexcept override; /// Operators Exception& operator=(const Exception&) = default; Exception& operator=(Exception&&) = default; const char* what() const noexcept override { return what_.c_str(); } virtual bool retryOnServer() const { return false; } virtual bool retryOnClient() const { return false; } virtual bool terminateApplication() const { return false; } static bool throwing(); static void exceptionStack(std::ostream&, bool callStack = false); const std::string& callStack() const { return callStack_; } const CodeLocation& location() const { return location_; } std::ostream& dumpStackTrace(std::ostream& = std::cout); protected: void reason(const std::string&); virtual void print(std::ostream&) const; private: std::string what_; ///< description std::string callStack_; ///< call stack SavedStatus save_; ///< saved monitor status to recover after destruction Exception* next_; CodeLocation location_; ///< where exception was first thrown friend std::ostream& operator<<(std::ostream& s, const Exception& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- class SeriousBug : public Exception { public: explicit SeriousBug(const std::string&, const CodeLocation& = {}); }; class TooManyRetries : public Exception { public: explicit TooManyRetries(int, const std::string& msg = ""); }; class TimeOut : public Exception { public: TimeOut(const std::string&, unsigned long); }; class FailedLibraryCall : public Exception { public: FailedLibraryCall(const std::string& lib, const std::string& func, const std::string& msg, const CodeLocation&); }; class FailedSystemCall : public Exception { public: FailedSystemCall(const std::string&, const CodeLocation& = {}, int = 0); }; class AssertionFailed : public Exception { public: explicit AssertionFailed(const std::string&, const CodeLocation& = {}); }; class BadParameter : public Exception { public: explicit BadParameter(const std::string&, const CodeLocation& = {}); }; class BadCast : public Exception { public: explicit BadCast(const std::string&, const CodeLocation& = {}); }; class BadValue : public Exception { public: explicit BadValue(const std::string&, const CodeLocation& = {}); }; class Stop : public Exception { public: explicit Stop(const std::string&, const CodeLocation& = {}); }; class Abort : public Exception { public: explicit Abort(const std::string&, const CodeLocation& = {}); }; class Cancel : public Exception { public: explicit Cancel(const std::string&, const CodeLocation& = {}); }; class Retry : public Exception { public: explicit Retry(const std::string&, const CodeLocation& = {}); }; class UserError : public Exception { public: explicit UserError(const std::string&, const CodeLocation& = {}); UserError(const std::string&, const std::string&, const CodeLocation& = {}); }; class OutOfRange : public Exception { public: explicit OutOfRange(const std::string&, const CodeLocation& = {}); OutOfRange(unsigned long long, unsigned long long, const CodeLocation& = {}); }; //---------------------------------------------------------------------------------------------------------------------- class NotImplemented : public Exception { public: explicit NotImplemented(const std::string&, const CodeLocation&); explicit NotImplemented(const CodeLocation& loc) : NotImplemented({}, loc) {} }; class FunctionalityNotSupported : public NotImplemented { public: explicit FunctionalityNotSupported(const std::string& what, const CodeLocation& = {}); }; //---------------------------------------------------------------------------------------------------------------------- // File errors class FileError : public Exception { protected: FileError(const std::string& file, const CodeLocation&); FileError() = default; }; class CantOpenFile : public FileError { bool retry_; bool retryOnServer() const override { return retry_; } public: explicit CantOpenFile(const std::string& file, bool retry = false, const CodeLocation& = {}); }; class WriteError : public FileError { public: explicit WriteError(const std::string& file, const CodeLocation& = {}); }; class ReadError : public FileError { public: explicit ReadError(const std::string& file, const CodeLocation& = {}); }; class CloseError : public FileError { public: explicit CloseError(const std::string& file, const CodeLocation& = {}); }; class ShortFile : public ReadError { public: explicit ShortFile(const std::string& file, const CodeLocation& = {}); }; //---------------------------------------------------------------------------------------------------------------------- class RemoteException : public Exception { public: RemoteException(const std::string&, const std::string& from); }; class UnexpectedState : public Exception { public: explicit UnexpectedState(const std::string&, const CodeLocation& = {}); }; //---------------------------------------------------------------------------------------------------------------------- template inline T SysCall(T code, const char* msg, const char* file, int line, const char* func) { if (code < 0) { throw FailedSystemCall(msg, CodeLocation(file, line, func), errno); } return code; } template inline long long SysCall(long long code, const char* msg, const T& ctx, const char* file, int line, const char* func) { if (code < 0) { std::ostringstream os; os << msg << " [" << ctx << "]"; throw FailedSystemCall(os.str(), CodeLocation(file, line, func), errno); } return code; } inline void ThrCall(int code, const char* msg, const char* file, int line, const char* func) { if (code != 0) { // Threads return errno in return code handle_panic(msg, CodeLocation(file, line, func)); } } /** * This functions hides that assertions may be handled by a throw of AssertionFailed * which is incompatible with using ASSERT inside destructors that are marked as no-throw, * which is the default behavior in C++11. * If env var ECKIT_ASSERT_ABORTS is set the assertion will abort instead of throwing. * In the future this will become the default behavior. */ inline void Assert(int code, const char* msg, const char* file, int line, const char* func) { if (code != 0) { handle_assert(msg, eckit::CodeLocation(file, line, func)); } } /** * This functions hides that assertions may be handled by a throw of AssertionFailed * which is incompatible with using ASSERT inside destructors that are marked as no-throw, * which is the default behavior in C++11. * If env var ECKIT_ASSERT_ABORTS is set the assertion will abort instead of throwing. * In the future this will become the default behavior. */ inline void Assert(int code, const std::string& msg, const char* file, int line, const char* func) { if (code != 0) { handle_assert(msg, eckit::CodeLocation(file, line, func)); } } inline void Panic(int code, const char* msg, const CodeLocation& loc) { if (code != 0) { handle_panic(msg, loc); } } inline void PanicNoLog(int code, const char* msg, const CodeLocation& loc) { if (code != 0) { handle_panic_no_log(msg, loc); } } //---------------------------------------------------------------------------------------------------------------------- // For compatibility class OutOfMemory : public Exception { bool terminateApplication() const override { return true; } const char* what() const noexcept override { return "OutOfMemory"; } public: OutOfMemory(); }; //---------------------------------------------------------------------------------------------------------------------- #define THRCALL(a) ::eckit::ThrCall(a, #a, __FILE__, __LINE__, __func__) #define SYSCALL(a) ::eckit::SysCall(a, #a, __FILE__, __LINE__, __func__) #define SYSCALL2(a, b) ::eckit::SysCall(a, #a, b, __FILE__, __LINE__, __func__) #define PANIC(a) ::eckit::Panic((a), #a, Here()) #define NOTIMP throw ::eckit::NotImplemented(Here()) #define ASSERT(a) static_cast(0), (a) ? (void)0 : ::eckit::Assert(!(a), #a, __FILE__, __LINE__, __func__) #define ASSERT_MSG(a, m) static_cast(0), (a) ? (void)0 : ::eckit::Assert(!(a), m, __FILE__, __LINE__, __func__) #define CHECK_CALL_NOLOG(a) ::eckit::PanicNoLog(a, #a, Here()) //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/exception/Exceptions.cc0000664000175000017500000002263415161702250021325 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/os/BackTrace.h" #include "eckit/thread/ThreadSingleton.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- static bool getenv_on(const char* env) { const auto* var = std::getenv(env); return var != nullptr && std::string(var) != "0"; }; static Exception*& first() { static ThreadSingleton p; return p.instance(); } Exception::Exception() : next_(first()) { first() = this; callStack_ = BackTrace::dump(); if (getenv_on("ECKIT_EXCEPTION_DUMPS_BACKTRACE")) { std::cerr << "Exception dumping backtrace: " << callStack_ << std::endl; } } Exception::~Exception() noexcept { first() = next_; } void Exception::print(std::ostream& out) const { out << what_; } void Exception::exceptionStack(std::ostream& out, bool callStack) { out << "Exception stack: " << std::endl; for (auto* e = first(); e != nullptr; e = e->next_) { out << e->what() << std::endl; if (callStack) { out << e->callStack() << std::endl << std::endl; } } out << "End stack" << std::endl; } Exception::Exception(const std::string& w, const CodeLocation& loc) : Exception(w, loc, false) {} Exception::Exception(const std::string& w, const CodeLocation& loc, bool quiet) : what_(w), next_(first()), location_(loc) { callStack_ = BackTrace::dump(); if (getenv_on("ECKIT_EXCEPTION_DUMPS_BACKTRACE")) { std::cerr << "Exception dumping backtrace: " << callStack_ << std::endl; } if (!getenv_on("ECKIT_EXCEPTION_IS_SILENT") && !quiet) { Log::error() << "Exception: " << w << " " << location_ << std::endl; Log::status() << "** " << w << location_ << std::endl; } first() = this; } std::ostream& Exception::dumpStackTrace(std::ostream& out) { return out << "Exception dumping backtrace: " << callStack_ << std::endl; } void Exception::reason(const std::string& w) { if (!getenv_on("ECKIT_EXCEPTION_IS_SILENT")) { Log::error() << "Exception: " << w << std::endl; } what_ = w; } bool Exception::throwing() { return first() != nullptr; } TooManyRetries::TooManyRetries(int retries, const std::string& w) : Exception("Too many retries: " + std::to_string(retries) + " @ " + w) { Log::status() << what() << std::endl; } TimeOut::TimeOut(const std::string& w, unsigned long timeout) : Exception("Timeout: " + std::to_string(timeout) + " (" + w + ")") {} FailedSystemCall::FailedSystemCall(const std::string& w, const CodeLocation& loc, int err) { errno = err; std::ostringstream s; s << "Failed system call: " << w << loc << Log::syserr; reason(s.str()); Log::status() << what() << std::endl; } SeriousBug::SeriousBug(const std::string& w, const CodeLocation& loc) : Exception("Serious bug: " + w, loc) { if (!getenv_on("ECKIT_SERIOUS_BUG_IS_SILENT")) { std::cout << what() << std::endl << BackTrace::dump() << std::endl; } } AssertionFailed::AssertionFailed(const std::string& w, const CodeLocation& loc) : Exception("Assertion failed: " + w, loc) {} void handle_assert(const std::string& msg, const CodeLocation& loc) { std::ostringstream s; s << "Assertion failed: " << msg << loc; if (!getenv_on("ECKIT_ASSERT_FAILED_IS_SILENT")) { Log::status() << s.str() << std::endl; std::cout << s.str() << std::endl << BackTrace::dump() << std::endl; } if (getenv_on("ECKIT_ASSERT_ABORTS")) { LibEcKit::instance().abort(); return; } throw AssertionFailed(msg, loc); } BadParameter::BadParameter(const std::string& w, const CodeLocation& loc) : Exception("Bad parameter: " + w, loc) {} BadCast::BadCast(const std::string& w, const CodeLocation& loc) : Exception("Bad cast: " + w, loc) {} BadValue::BadValue(const std::string& w, const CodeLocation& loc) : Exception("Bad value: " + w, loc) {} Stop::Stop(const std::string& w, const CodeLocation& loc) : Exception("Stop: " + w, loc) {} Abort::Abort(const std::string& w, const CodeLocation& loc) : Exception("Abort: " + w, loc) {} Cancel::Cancel(const std::string& w, const CodeLocation& loc) : Exception("Cancel: " + w, loc) {} Retry::Retry(const std::string& w, const CodeLocation& loc) : Exception("Retry: " + w, loc) {} UserError::UserError(const std::string& w, const CodeLocation& loc) : Exception("UserError: " + w, loc) {} UserError::UserError(const std::string& user, const std::string& w, const CodeLocation& loc) : UserError(user + " : " + w, loc) {} OutOfRange::OutOfRange(unsigned long long index, unsigned long long max, const CodeLocation& loc) : Exception( "Out of range: accessing element " + std::to_string(index) + ", but maximum is " + std::to_string(max - 1), loc) {} OutOfRange::OutOfRange(const std::string& w, const CodeLocation& loc) : Exception("Out of range: " + w, loc) {} //---------------------------------------------------------------------------------------------------------------------- NotImplemented::NotImplemented(const std::string& w, const eckit::CodeLocation& loc) : Exception("Not implemented" + (w.empty() ? "" : (": " + w)), loc) { Log::status() << what() << std::endl; std::cout << what() << std::endl << BackTrace::dump() << std::endl; } FunctionalityNotSupported::FunctionalityNotSupported(const std::string& what, const CodeLocation& loc) : NotImplemented("Functionality not supported: " + what, loc) {} //---------------------------------------------------------------------------------------------------------------------- FileError::FileError(const std::string& w, const CodeLocation& loc) { std::ostringstream s; s << "Failed system call: " << w << loc << Log::syserr; reason(s.str()); Log::status() << what() << std::endl; } CantOpenFile::CantOpenFile(const std::string& file, bool retry, const CodeLocation& loc) : retry_(retry) { std::ostringstream s; s << "Cannot open " << file << " " << Log::syserr; if (retry) { s << " (retry ok)"; } s << loc; reason(s.str()); Log::status() << what() << std::endl; } WriteError::WriteError(const std::string& file, const CodeLocation& loc) : FileError("Write error on " + file, loc) {} ReadError::ReadError(const std::string& file, const CodeLocation& loc) : FileError("Read error on " + file, loc) {} CloseError::CloseError(const std::string& file, const CodeLocation& loc) : FileError("Close error on " + file, loc) {} ShortFile::ShortFile(const std::string& file, const CodeLocation& loc) : ReadError("Short file while reading " + file, loc) {} //---------------------------------------------------------------------------------------------------------------------- RemoteException::RemoteException(const std::string& w, const std::string& from) : Exception(w + "(RemoteException from " + from + ")") {} UnexpectedState::UnexpectedState(const std::string& w, const CodeLocation& loc) : Exception("Unexpected state: " + w, loc) {} //---------------------------------------------------------------------------------------------------------------------- void handle_panic(const char* msg, const CodeLocation& loc) { std::cout << "PANIC: " << (msg == nullptr ? "(null message)" : msg) << loc << std::endl; std::cerr << "PANIC: " << (msg == nullptr ? "(null message)" : msg) << loc << std::endl; std::cerr << "----------------------------------------\n" << "BACKTRACE\n" << "----------------------------------------\n" << BackTrace::dump() << std::endl << "----------------------------------------\n" << std::endl; if (getenv_on("STOP_ON_PANIC")) { pid_t pid = ::getpid(); std::cout << "Stopped process with PID " << pid << " - attach a debugger or send a SIGCONT signal to abort" << std::endl; std::cerr << "Stopped process with PID " << pid << " - attach a debugger or send a SIGCONT signal to abort" << std::endl; ::kill(pid, SIGSTOP); ::kill(pid, SIGABRT); } _exit(1); } void handle_panic_no_log(const char* msg, const CodeLocation& loc) { std::cout << "PANIC: " << (msg == nullptr ? "(null message)" : msg) << loc << std::endl; std::cerr << "PANIC: " << (msg == nullptr ? "(null message)" : msg) << loc << std::endl; ::kill(::getpid(), SIGABRT); ::pause(); } OutOfMemory::OutOfMemory() : Exception("out of memory") {} FailedLibraryCall::FailedLibraryCall(const std::string& lib, const std::string& func, const std::string& w, const CodeLocation& loc) : Exception(w, loc) { std::ostringstream s; s << "Failed function call " << func << " to library " << lib << " : " << w << " " << " @ " << loc; reason(s.str()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/CMakeLists.txt0000664000175000017500000005571715161702250017447 0ustar alastairalastair### endianess test_big_endian( _BIG_ENDIAN ) if( _BIG_ENDIAN ) set( eckit_BIG_ENDIAN 1 ) set( eckit_LITTLE_ENDIAN 0 ) else() set( eckit_BIG_ENDIAN 0 ) set( eckit_LITTLE_ENDIAN 1 ) endif() set( ECKIT_BIG_ENDIAN ${eckit_BIG_ENDIAN} ) set( ECKIT_LITTLE_ENDIAN ${eckit_LITTLE_ENDIAN} ) ### check support for DL library check_include_files( dlfcn.h eckit_HAVE_DLFCN_H ) if(NOT eckit_HAVE_DLFCN_H) message(FATAL_ERROR "eckit requires dlfcn.h for supporting dlopen of shared libraries") endif() cmake_push_check_state(RESET) set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_DL_LIBS} ) check_c_source_compiles( "#define _GNU_SOURCE\n#include \nint main(){ void* addr; Dl_info info; dladdr(addr, &info); }\n" eckit_HAVE_DLADDR ) cmake_pop_check_state() ### system headers and symbols tests for eckit config headers check_symbol_exists( MAP_ANONYMOUS "sys/mman.h" eckit_HAVE_MAP_ANONYMOUS ) check_symbol_exists( MAP_ANON "sys/mman.h" eckit_HAVE_MAP_ANON ) check_symbol_exists( funopen "stdio.h" eckit_HAVE_FUNOPEN ) check_symbol_exists( fsync "unistd.h" eckit_HAVE_FSYNC ) check_symbol_exists( fdatasync "unistd.h" eckit_HAVE_FDATASYNC ) check_symbol_exists( F_FULLFSYNC "fcntl.h" eckit_HAVE_F_FULLFSYNC ) check_symbol_exists( fmemopen "stdio.h" eckit_HAVE_FMEMOPEN ) check_symbol_exists( dlinfo "dlfcn.h" eckit_HAVE_DLINFO ) check_symbol_exists( feenableexcept "fenv.h" eckit_HAVE_FEENABLEEXCEPT ) check_symbol_exists( fedisableexcept "fenv.h" eckit_HAVE_FEDISABLEEXCEPT ) check_c_source_compiles( "#define _GNU_SOURCE\n#include \nint main(){ void* cookie; const char* mode; cookie_io_functions_t iof; FILE* fopencookie(void *cookie, const char *mode, cookie_io_functions_t iof); }" eckit_HAVE_FOPENCOOKIE ) check_c_source_compiles( "#include \n#include \n int main(){ void ** buffer; int i = backtrace(buffer, 256); }\n" eckit_HAVE_EXECINFO_BACKTRACE ) check_cxx_source_compiles( "#include \n int main() { char * type; int status; char * r = abi::__cxa_demangle(type, 0, 0, &status); }" eckit_HAVE_CXXABI_H ) check_c_source_compiles( "#include \nint main(){ time_t now; time(&now); struct tm t; gmtime_r(&now,&t); }\n" eckit_HAVE_GMTIME_R ) check_c_source_compiles( "#include \nint main(){ DIR *dirp; struct dirent *entry; struct dirent **result; int i = readdir_r(dirp, entry, result); }\n" eckit_HAVE_READDIR_R ) check_c_source_compiles( "#include \n#include \nint main(){ DIR *dirp; int i = dirfd(dirp); }\n" eckit_HAVE_DIRFD ) check_c_source_compiles( "#include \nint main(){ DIR *dirp; struct dirent *entry; if(entry->d_type) { dirp = 0; } }\n" eckit_HAVE_DIRENT_D_TYPE ) check_cxx_source_compiles( "int main() { __int128 i = 0; return 0;}" eckit_HAVE_CXX_INT_128 ) ### config headers ecbuild_generate_config_headers( DESTINATION ${INSTALL_INCLUDE_DIR}/eckit ) configure_file( eckit_config.h.in eckit_config.h ) configure_file( eckit_version.h.in eckit_version.h ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/eckit_config.h ${CMAKE_CURRENT_BINARY_DIR}/eckit_version.h DESTINATION ${INSTALL_INCLUDE_DIR}/eckit ) configure_file( eckit_version.cc.in eckit_version.cc ) ### eckit sources list( APPEND eckit_srcs eckit.h deprecated.h ${CMAKE_CURRENT_BINARY_DIR}/eckit_version.cc ) list( APPEND eckit_container_srcs container/BSPTree.h container/BTree.cc container/BTree.h container/BloomFilter.cc container/BloomFilter.h container/Cache.h container/CacheLRU.cc container/CacheLRU.h container/CacheManager.cc container/CacheManager.h container/ClassExtent.h container/DenseMap.h container/DenseSet.h container/KDMapped.cc container/KDMapped.h container/KDMemory.h container/KDTree.h container/MappedArray.cc container/MappedArray.h container/Queue.h container/Recycler.h container/SharedMemArray.cc container/SharedMemArray.h container/StatCollector.h container/Trie.cc container/Trie.h container/bsptree/BSPHyperPlane.h container/bsptree/BSPNode.cc container/bsptree/BSPNode.h container/kdtree/KDNode.cc container/kdtree/KDNode.h container/sptree/SPIterator.h container/sptree/SPMetadata.h container/sptree/SPNode.h container/sptree/SPNodeInfo.h container/sptree/SPNodeQueue.h container/sptree/SPTree.h container/sptree/SPValue.h ) list( APPEND eckit_io_srcs io/AIOHandle.cc io/AIOHandle.h io/AsyncHandle.cc io/AsyncHandle.h io/AutoCloser.h io/Base64.cc io/Base64.h io/BitIO.cc io/BitIO.h io/Buffer.cc io/Buffer.h io/BufferCache.cc io/BufferCache.h io/BufferList.cc io/BufferList.h io/BufferedHandle.cc io/BufferedHandle.h io/CircularBuffer.cc io/CircularBuffer.h io/CommandStream.cc io/CommandStream.h io/Compress.cc io/Compress.h io/DataHandle.cc io/DataHandle.h io/DblBuffer.cc io/DblBuffer.h io/EmptyHandle.cc io/EmptyHandle.h io/FDataSync.cc io/FDataSync.h io/FOpenDataHandle.cc io/FTPHandle.cc io/FTPHandle.h io/FileBase.cc io/FileBase.h io/FileDescHandle.cc io/FileDescHandle.h io/FileHandle.cc io/FileHandle.h io/FileLock.cc io/FileLock.h io/FileLocker.cc io/FileLocker.h io/FilePool.cc io/FilePool.h io/HandleBuf.cc io/HandleBuf.h io/HandleHolder.cc io/HandleHolder.h io/Length.cc io/Length.h io/MMappedFileHandle.cc io/MMappedFileHandle.h io/MemoryHandle.cc io/MemoryHandle.h io/MoverTransfer.cc io/MoverTransfer.h io/MoverTransferSelection.cc io/MoverTransferSelection.h io/MultiHandle.cc io/MultiHandle.h io/MultiSocketHandle.cc io/MultiSocketHandle.h io/Offset.cc io/Offset.h io/PartFileHandle.cc io/PartFileHandle.h io/PartHandle.cc io/PartHandle.h io/PeekHandle.cc io/PeekHandle.h io/PipeHandle.cc io/PipeHandle.h io/Pipeline.cc io/Pipeline.h io/PooledFile.cc io/PooledFile.h io/PooledFileDescriptor.cc io/PooledFileDescriptor.h io/PooledHandle.cc io/PooledHandle.h io/RawFileHandle.cc io/RawFileHandle.h io/ResizableBuffer.h io/SeekableHandle.cc io/SeekableHandle.h io/Select.cc io/Select.h io/SharedBuffer.cc io/SharedBuffer.h io/SharedHandle.cc io/SharedHandle.h io/SockBuf.cc io/SockBuf.h io/StatsHandle.cc io/StatsHandle.h io/StdFile.cc io/StdFile.h io/StdPipe.cc io/StdPipe.h io/StdioBuf.cc io/StdioBuf.h io/TCPHandle.cc io/TCPHandle.h io/TCPSocketHandle.cc io/TCPSocketHandle.h io/TeeHandle.cc io/TeeHandle.h io/TransferWatcher.cc io/TransferWatcher.h io/cluster/ClusterDisks.cc io/cluster/ClusterDisks.h io/cluster/ClusterNode.cc io/cluster/ClusterNode.h io/cluster/ClusterNodes.cc io/cluster/ClusterNodes.h io/cluster/NodeInfo.cc io/cluster/NodeInfo.h ) if(HAVE_CURL) list(APPEND eckit_io_srcs io/EasyCURL.cc io/EasyCURL.h io/URLHandle.cc io/URLHandle.h ) endif() list(APPEND eckit_message_srcs message/Decoder.cc message/Decoder.h message/Message.cc message/Message.h message/MessageContent.cc message/MessageContent.h message/Reader.cc message/Reader.h message/Splitter.cc message/Splitter.h ) if(HAVE_RADOS) list( APPEND eckit_io_srcs io/rados/RadosHandle.cc io/rados/RadosHandle.h io/rados/RadosCluster.h io/rados/RadosCluster.cc io/rados/RadosReadHandle.cc io/rados/RadosReadHandle.h io/rados/RadosWriteHandle.cc io/rados/RadosWriteHandle.h io/rados/RadosObject.h io/rados/RadosObject.cc io/rados/RadosAttributes.h io/rados/RadosAttributes.cc ) endif() list( APPEND eckit_filesystem_srcs filesystem/BasePathName.cc filesystem/BasePathName.h filesystem/BasePathNameT.cc filesystem/BasePathNameT.h filesystem/FileMode.cc filesystem/FileMode.h filesystem/FileSpace.cc filesystem/FileSpace.h filesystem/FileSpaceStrategies.cc filesystem/FileSpaceStrategies.h filesystem/FileSystem.cc filesystem/FileSystem.h filesystem/FileSystemSize.h filesystem/LocalPathName.cc filesystem/LocalPathName.h filesystem/PathExpander.cc filesystem/PathExpander.h filesystem/PathName.cc filesystem/PathName.h filesystem/PathNameFactory.cc filesystem/PathNameFactory.h filesystem/TempFile.cc filesystem/TempFile.h filesystem/TmpDir.cc filesystem/TmpDir.h filesystem/StdDir.cc filesystem/StdDir.h filesystem/TmpFile.cc filesystem/TmpFile.h filesystem/URI.cc filesystem/URI.h filesystem/URIManager.cc filesystem/URIManager.h filesystem/LocalFileManager.cc filesystem/LocalFileManager.h ) list( APPEND eckit_thread_srcs thread/AutoLock.h thread/Mutex.cc thread/Mutex.h thread/MutexCond.cc thread/MutexCond.h thread/Once.h thread/StaticMutex.cc thread/StaticMutex.h thread/Thread.cc thread/Thread.h thread/ThreadControler.cc thread/ThreadControler.h thread/ThreadPool.cc thread/ThreadPool.h thread/ThreadSingleton.h ) list( APPEND eckit_config_srcs config/Configurable.cc config/Configurable.h config/Configuration.cc config/Configuration.h config/Configured.cc config/Configured.h config/EtcTable.cc config/EtcTable.h config/JSONConfiguration.h config/LibEcKit.cc config/LibEcKit.h config/LocalConfiguration.cc config/LocalConfiguration.h config/Parametrisation.cc config/Parametrisation.h config/Resource.h config/ResourceBase.cc config/ResourceBase.h config/ResourceMgr.cc config/ResourceMgr.h config/YAMLConfiguration.cc config/YAMLConfiguration.h ) list( APPEND eckit_runtime_srcs runtime/Application.cc runtime/Application.h runtime/Dispatcher.h runtime/Library.cc runtime/Library.h runtime/Main.cc runtime/Main.h runtime/Metrics.cc runtime/Metrics.h runtime/Monitor.cc runtime/Monitor.h runtime/Monitorable.cc runtime/Monitorable.h runtime/Pipe.h runtime/PipeApplication.cc runtime/PipeApplication.h runtime/PipeHandler.cc runtime/PipeHandler.h runtime/ProcessControler.cc runtime/ProcessControler.h runtime/ProducerConsumer.h runtime/Telemetry.cc runtime/Telemetry.h runtime/SessionID.cc runtime/SessionID.h runtime/Task.cc runtime/Task.h runtime/TaskID.h runtime/TaskInfo.cc runtime/TaskInfo.h runtime/Tool.cc runtime/Tool.h ) list( APPEND eckit_log_srcs log/BigNum.cc log/BigNum.h log/Bytes.cc log/Bytes.h log/CallbackTarget.cc log/CallbackTarget.h log/Channel.cc log/Channel.h log/ChannelBuffer.cc log/ChannelBuffer.h log/CodeLocation.cc log/CodeLocation.h log/Colour.cc log/Colour.h log/ColouringTarget.cc log/ColouringTarget.h log/ETA.cc log/ETA.h log/FileTarget.cc log/FileTarget.h log/IndentTarget.cc log/IndentTarget.h log/JSON.cc log/JSON.h log/LineBasedTarget.cc log/LineBasedTarget.h log/Log.cc log/Log.h log/LogTarget.cc log/LogTarget.h log/MessageTarget.cc log/MessageTarget.h log/MonitorTarget.cc log/MonitorTarget.h log/Number.cc log/Number.h log/OStreamTarget.cc log/OStreamTarget.h log/Plural.h log/PrefixTarget.cc log/PrefixTarget.h log/Progress.cc log/Progress.h log/ProgressTimer.cc log/ProgressTimer.h log/ResourceUsage.cc log/ResourceUsage.h log/RotationTarget.cc log/RotationTarget.h log/SavedStatus.cc log/SavedStatus.h log/Seconds.cc log/Seconds.h log/Statistics.cc log/Statistics.h log/StatusTarget.cc log/StatusTarget.h log/SysLog.cc log/SysLog.h log/SysLogTCPTarget.cc log/SysLogTCPTarget.h log/TCPTarget.cc log/TCPTarget.h log/TeeTarget.cc log/TeeTarget.h log/TimeStamp.cc log/TimeStamp.h log/TimeStampTarget.cc log/TimeStampTarget.h log/Timer.cc log/Timer.h log/TraceTimer.h log/UserChannel.cc log/UserChannel.h log/WrapperTarget.cc log/WrapperTarget.h ) list( APPEND eckit_exception_srcs exception/Exceptions.cc exception/Exceptions.h ) list( APPEND eckit_types_srcs types/ClimateDate.cc types/ClimateDate.h types/Coord.cc types/Coord.h types/Date.cc types/Date.h types/DateTime.cc types/DateTime.h types/DayOfYear.cc types/DayOfYear.h types/Double.cc types/Double.h types/FixedString.h types/FloatCompare.cc types/FloatCompare.h types/Fraction.cc types/Fraction.h types/Grid.cc types/Grid.h types/Hour.cc types/Hour.h types/Month.cc types/Month.h types/SemanticVersion.cc types/SemanticVersion.h types/Time.cc types/Time.h types/TimeInterval.cc types/TimeInterval.h types/Types.cc types/Types.h types/UUID.cc types/UUID.h types/VerifyingDate.cc types/VerifyingDate.h ) list( APPEND eckit_parser_srcs parser/CSVParser.cc parser/CSVParser.h parser/JSONParser.cc parser/JSONParser.h parser/ObjectParser.cc parser/ObjectParser.h parser/StreamParser.cc parser/StreamParser.h parser/YAMLParser.cc parser/YAMLParser.h ) list( APPEND eckit_value_srcs value/BoolContent.cc value/BoolContent.h value/CompositeParams.cc value/CompositeParams.h value/Content.cc value/Content.h value/DateContent.cc value/DateContent.h value/DateTimeContent.cc value/DateTimeContent.h value/DispatchParams.h value/DoubleContent.cc value/DoubleContent.h value/Expression.h value/ListContent.cc value/ListContent.h value/MapContent.cc value/MapContent.h value/NilContent.cc value/NilContent.h value/NumberContent.cc value/NumberContent.h value/OrderedMapContent.cc value/OrderedMapContent.h value/Params.cc value/Params.h value/Properties.cc value/Properties.h value/ScopeParams.cc value/ScopeParams.h value/StringContent.cc value/StringContent.h value/TimeContent.cc value/TimeContent.h value/Value.cc value/Value.h ) list( APPEND eckit_os_srcs os/AutoAlarm.cc os/AutoAlarm.h os/AutoUmask.h os/BackTrace.cc os/BackTrace.h os/Password.cc os/Password.h os/SemLocker.cc os/SemLocker.h os/Semaphore.cc os/Semaphore.h os/SharedInt.cc os/SharedInt.h os/SignalHandler.cc os/SignalHandler.h os/Stat.h os/System.cc os/System.h ) list( APPEND eckit_net_srcs net/Connector.cc net/Connector.h net/Endpoint.cc net/Endpoint.h net/HttpHeader.cc net/HttpHeader.h net/IPAddress.cc net/IPAddress.h net/MultiSocket.cc net/MultiSocket.h net/NetMask.cc net/NetMask.h net/NetService.cc net/NetService.h net/NetUser.cc net/NetUser.h net/Port.cc net/Port.h net/ProxiedTCPClient.cc net/ProxiedTCPClient.h net/ProxiedTCPServer.cc net/ProxiedTCPServer.h net/SocketOptions.cc net/SocketOptions.h net/TCPClient.cc net/TCPClient.h net/TCPServer.cc net/TCPServer.h net/TCPSocket.cc net/TCPSocket.h net/TCPStream.cc net/TCPStream.h net/Telnet.cc net/Telnet.h net/TelnetUser.cc net/TelnetUser.h net/Telnetable.cc net/Telnetable.h net/UDPClient.cc net/UDPClient.h net/UDPServer.cc net/UDPServer.h ) list( APPEND eckit_serialisation_srcs serialisation/BadTag.cc serialisation/BadTag.h serialisation/FileStream.cc serialisation/FileStream.h serialisation/FstreamStream.h serialisation/HandleStream.h serialisation/IfstreamStream.h serialisation/MemoryStream.cc serialisation/MemoryStream.h serialisation/PipeStream.cc serialisation/PipeStream.h serialisation/Reanimator.cc serialisation/Reanimator.h serialisation/ReanimatorBase.cc serialisation/ResizableMemoryStream.cc serialisation/ResizableMemoryStream.h serialisation/Stream.cc serialisation/Stream.h serialisation/Streamable.cc serialisation/Streamable.h ) list( APPEND eckit_persist_srcs persist/Bless.h persist/DumpLoad.cc persist/DumpLoad.h persist/Exporter.cc persist/Exporter.h persist/Isa.cc persist/Isa.h ) list( APPEND eckit_utils_srcs utils/ByteSwap.h utils/Clock.h utils/Compressor.cc utils/Compressor.h utils/EnumBitmask.h utils/Hash.cc utils/Hash.h utils/HyperCube.cc utils/HyperCube.h utils/Literals.h utils/MD5.cc utils/MD5.h utils/Overloaded.h utils/RLE.cc utils/RLE.h utils/Regex.cc utils/Regex.h utils/RendezvousHash.cc utils/RendezvousHash.h utils/SafeCasts.h utils/StringTools.cc utils/StringTools.h utils/Tokenizer.cc utils/Tokenizer.h utils/Translator.cc utils/Translator.h ) if(eckit_HAVE_BZIP2) list( APPEND eckit_utils_srcs utils/BZip2Compressor.cc utils/BZip2Compressor.h ) endif() if(eckit_HAVE_SNAPPY) list( APPEND eckit_utils_srcs utils/SnappyCompressor.cc utils/SnappyCompressor.h ) endif() if(eckit_HAVE_LZ4) list( APPEND eckit_utils_srcs utils/LZ4Compressor.cc utils/LZ4Compressor.h ) endif() if(eckit_HAVE_AEC) list( APPEND eckit_utils_srcs utils/AECCompressor.cc utils/AECCompressor.h ) endif() if(eckit_HAVE_SSL) list( APPEND eckit_utils_srcs utils/MD4.cc utils/MD4.h utils/SHA1.cc utils/SHA1.h ) endif() if(eckit_HAVE_XXHASH) list( APPEND eckit_utils_srcs utils/xxHashing.cc utils/xxHashing.h contrib/xxhash/xxhash.h ) if( CMAKE_CXX_COMPILER_ID MATCHES PGI|NVHPC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 21.9 ) # ECKIT-574: work around missing reference to "__builtin_rotateleft64" set_source_files_properties(utils/xxHashing.cc PROPERTIES COMPILE_FLAGS -DNO_CLANG_BUILTIN) endif() endif() list( APPEND eckit_memory_srcs memory/Builder.h memory/Counted.cc memory/Counted.h memory/Factory.h memory/MMap.cc memory/MMap.h memory/MapAllocator.cc memory/MapAllocator.h memory/MemoryBuffer.cc memory/MemoryBuffer.h memory/NonCopyable.cc memory/NonCopyable.h memory/OnlyMovable.h memory/Owned.h memory/Padded.h memory/Shmget.cc memory/Shmget.h memory/Zero.h ) list( APPEND eckit_compat_srcs compat/Inited.h compat/StrStream.h ) list( APPEND eckit_maths_srcs maths/Functions.cc maths/Functions.h ) list( APPEND eckit_system_srcs system/Library.cc system/Library.h system/LibraryManager.cc system/LibraryManager.h system/MemoryInfo.cc system/MemoryInfo.h system/Plugin.cc system/Plugin.h system/ResourceUsage.cc system/ResourceUsage.h system/SystemInfo.cc system/SystemInfo.h ) set(ECKIT_SYSTEM_EXTRA_LIBS "") if( EC_OS_NAME STREQUAL "linux" ) list( APPEND eckit_system_srcs system/SystemInfoLinux.h system/SystemInfoLinux.cc ) elseif( EC_OS_NAME STREQUAL "freebsd" ) list( APPEND eckit_system_srcs system/SystemInfoFreeBSD.h system/SystemInfoFreeBSD.cc ) list( APPEND ECKIT_SYSTEM_EXTRA_LIBS "execinfo" ) # Add in support for backtrace on FreeBSD elseif( EC_OS_NAME STREQUAL "macosx" ) list( APPEND eckit_system_srcs system/SystemInfoMacOSX.h system/SystemInfoMacOSX.cc ) else() ecbuild_warn("eckit does not support SystemInfo for OS '${EC_OS_NAME}'") endif() list( APPEND eckit_bases_srcs bases/Loader.cc bases/Loader.h bases/Watcher.cc bases/Watcher.h ) list( APPEND eckit_transaction_srcs transaction/TxnEvent.cc transaction/TxnEvent.h transaction/TxnLog.cc transaction/TxnLog.h ) list( APPEND eckit_testing_srcs testing/Filesystem.cc testing/Filesystem.h testing/Test.h testing/Filesystem.cc testing/Filesystem.h ) list( APPEND eckit_dirs bases compat config container exception filesystem io log memory message net os parser persist runtime serialisation thread transaction value maths system testing types utils ) foreach( dir ${eckit_dirs} ) list( APPEND eckit_srcs ${eckit_${dir}_srcs} ) endforeach() list( APPEND eckit_templates container/BTree.cc container/BloomFilter.cc container/CacheLRU.cc container/MappedArray.cc container/SharedMemArray.cc container/Trie.cc container/bsptree/BSPNode.cc container/kdtree/KDNode.cc container/sptree/SPNode.cc filesystem/BasePathNameT.cc io/FileBase.cc runtime/PipeHandler.cc serialisation/Reanimator.cc transaction/TxnLog.cc types/Types.cc utils/RLE.cc ) list( APPEND eckit_persistent io/Length.h io/Offset.h types/ClimateDate.h types/Date.h types/DateTime.h types/DayOfYear.h types/Double.h types/Grid.h types/Month.h types/Time.h types/VerifyingDate.h ) ### eckit library ecbuild_add_library( TARGET eckit TYPE SHARED INSTALL_HEADERS LISTED HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit SOURCES ${eckit_srcs} TEMPLATES ${eckit_templates} PERSISTENT ${eckit_persistent} PUBLIC_INCLUDES $ $ PRIVATE_INCLUDES "${CURL_INCLUDE_DIRS}" "${SNAPPY_INCLUDE_DIRS}" "${LZ4_INCLUDE_DIRS}" "${BZIP2_INCLUDE_DIRS}" "${RADOS_INCLUDE_DIRS}" "${OPENSSL_INCLUDE_DIR}" "${AIO_INCLUDE_DIRS}" PRIVATE_LIBS "${SNAPPY_LIBRARIES}" "${LZ4_LIBRARIES}" "${BZIP2_LIBRARIES}" "${OPENSSL_LIBRARIES}" "${CURL_LIBRARIES}" "${AIO_LIBRARIES}" "${RADOS_LIBRARIES}" $<${HAVE_AEC}:libaec::aec> PUBLIC_LIBS ${CMATH_LIBRARIES} ${RT_LIBRARIES} ${THREADS_LIBRARIES} ${CMAKE_DL_LIBS} ${ECKIT_SYSTEM_EXTRA_LIBS} LINKER_LANGUAGE CXX ) ### sub-directories if( eckit_HAVE_ECKIT_CMD ) add_subdirectory( cmd ) endif() if( eckit_HAVE_ECKIT_SQL ) add_subdirectory( sql ) endif() add_subdirectory( distributed ) add_subdirectory( geometry ) add_subdirectory( linalg ) add_subdirectory( maths ) add_subdirectory( mpi ) add_subdirectory( option ) add_subdirectory( web ) if( eckit_HAVE_ECKIT_CODEC ) add_subdirectory( codec ) endif() if( eckit_HAVE_ECKIT_SPEC ) add_subdirectory( spec ) endif() if( eckit_HAVE_ECKIT_GEO ) add_subdirectory( geo ) endif() eckit-2.0.7/src/eckit/net/0000775000175000017500000000000015161702250015456 5ustar alastairalastaireckit-2.0.7/src/eckit/net/TCPServer.cc0000664000175000017500000000712215161702250017604 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include "eckit/config/Resource.h" #include "eckit/io/Select.h" #include "eckit/log/Log.h" #include "eckit/net/TCPServer.h" #include "eckit/thread/AutoLock.h" namespace eckit::net { TCPServer::TCPServer(const SocketOptions& options) : TCPSocket(), port_(0), listen_(-1), options_(options), closeExec_(true) {} TCPServer::TCPServer(int port, const SocketOptions& options) : TCPSocket(), port_(port), listen_(-1), options_(options), closeExec_(true) {} TCPServer::~TCPServer() { if (listen_ >= 0) { ::close(listen_); } } TCPSocket& TCPServer::accept(const std::string& message, int timeout, bool* connected) { bind(); sockaddr_in from; socklen_t fromlen = sizeof(from); for (;;) { int delay = timeout ? timeout : 10; Select select(listen_); Log::status() << message; if (port_) { Log::status() << " (port " << port_ << ")"; } Log::status() << std::endl; while (!select.ready(delay)) { if (timeout && !connected) { throw TimeOut(message, timeout); } if (connected) { *connected = false; return *this; } Log::status() << message; if (port_) { Log::status() << " (port " << port_ << ")"; } Log::status() << std::endl; } if ((socket_ = ::accept(listen_, reinterpret_cast(&from), &fromlen)) >= 0) { break; } if (errno != EINTR) { throw FailedSystemCall("accept"); } } remoteAddr_ = from.sin_addr; remoteHost_ = addrToHost(from.sin_addr); remotePort_ = ntohs(from.sin_port); // Set the 'close on exec' if (closeExec_) { SYSCALL(fcntl(socket_, F_SETFD, FD_CLOEXEC)); } register_ignore_sigpipe(); Log::status() << "Get connection from " << remoteHost() << std::endl; if (connected) { *connected = true; } return *this; } void TCPServer::close() { TCPSocket::close(); if (listen_ >= 0) { ::close(listen_); } listen_ = -1; } void TCPServer::bind() { // There can be a race condition is two thread asks for localPort() // and both try to bind at the same time AutoLock lock(mutex_); if (listen_ == -1) { listen_ = createSocket(port_, options_); int backlog = options_.listenBacklog(); Log::info() << "Listening on socket " << listen_ << " port: " << port_ << " backlog: " << backlog << std::endl; SYSCALL(::listen(listen_, backlog)); } } int TCPServer::socket() { bind(); return listen_; } std::string TCPServer::bindingAddress() const { return options_.bindAddress(); } void TCPServer::print(std::ostream& s) const { s << "TCPServer[" << "port=" << port_ << ",options_=" << options_ << ","; TCPSocket::print(s); s << "]"; } EphemeralTCPServer::EphemeralTCPServer(const SocketOptions& opts) : TCPServer(0, opts) {} EphemeralTCPServer::EphemeralTCPServer(int port, const SocketOptions& opts) : TCPServer(port, opts) {} } // namespace eckit::net eckit-2.0.7/src/eckit/net/NetService.h0000664000175000017500000000274115161702250017702 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Jun 96 #ifndef eckit_NetService_h #define eckit_NetService_h #include "eckit/net/TCPServer.h" #include "eckit/thread/Thread.h" namespace eckit::net { class NetUser; class NetService : public Thread { public: /// @param[in] port TCP port to listen on /// @param[in] visible Make the thread this service is running in visible on the Monitor (defaults to false) NetService(int port, bool visible = true, const SocketOptions& options = SocketOptions::server()); ~NetService(); /// @returns hostname to which this server is answering std::string hostname() const; /// @returns port to which this server is answering int port() const; void run() override; private: TCPServer server_; bool visible_; ///< Visible on the Monitor? private: virtual NetUser* newUser(net::TCPSocket&) const = 0; virtual std::string name() const = 0; virtual bool preferToRunAsProcess() const; virtual bool runAsProcess() const; virtual long timeout() const; }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/TCPSocket.cc0000664000175000017500000005411215161702250017567 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // Disable warnings: warning #550-D: variable "__d0" was set but never used [set_but_not_used] in FD_ZERO(&r); #if defined(__NVCOMPILER) #pragma diag_suppress 550 #endif #include // FreeBSD: must appear before netinet/ip.h #include #include #include #include #include #include #include #include #include #include #include #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/Select.h" #include "eckit/log/Log.h" #include "eckit/log/Seconds.h" #include "eckit/memory/Zero.h" #include "eckit/net/IPAddress.h" #include "eckit/net/TCPClient.h" #include "eckit/net/TCPSocket.h" #include "eckit/os/AutoAlarm.h" #include "eckit/runtime/Main.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" namespace eckit::net { static in_addr none = {INADDR_NONE}; static StaticMutex local_mutex; TCPSocket::UnknownHost::UnknownHost(const std::string& host) : Exception(std::string("Unknown host ") + host) {} TCPSocket::TCPSocket() : socket_(-1), localPort_(-1), remotePort_(-1), remoteAddr_(none), localAddr_(none) {} // This contructor performs a change of ownership of the socket TCPSocket::TCPSocket(net::TCPSocket& other) : socket_(other.socket_), localPort_(other.localPort_), remotePort_(other.remotePort_), remoteHost_(other.remoteHost_), remoteAddr_(other.remoteAddr_), localHost_(other.localHost_), localAddr_(other.localAddr_) { other.socket_ = -1; // Detach socket from other other.remoteAddr_ = none; other.remoteHost_ = std::string(); other.remotePort_ = -1; } TCPSocket::~TCPSocket() { close(); } // This contructor performs a change of ownership of the socket TCPSocket& TCPSocket::operator=(net::TCPSocket& other) { socket_ = other.socket_; localAddr_ = other.localAddr_; localHost_ = other.localHost_; localPort_ = other.localPort_; remoteAddr_ = other.remoteAddr_; remoteHost_ = other.remoteHost_; remotePort_ = other.remotePort_; debug_ = other.debug_; other.socket_ = -1; // Detach socket from other other.remoteAddr_ = none; other.remoteHost_ = std::string(); other.remotePort_ = -1; return *this; } void TCPSocket::closeOutput() { SYSCALL(::shutdown(socket_, SHUT_WR)); } void TCPSocket::closeInput() { SYSCALL(::shutdown(socket_, SHUT_RD)); } long TCPSocket::write(const void* buf, long length) const { // Allow zero length packets if (length == 0) { return ::write(socket_, buf, length); } long requested = length; if (debug_.on) { if (debug_.mode != 'w') { debug_.newline = true; std::cout << std::endl << std::endl; debug_.mode = 'w'; } const char* p = reinterpret_cast(buf); for (long i = 0; i < std::min(length, 512L); i++) { if (debug_.newline) { std::cout << ">>> "; debug_.newline = false; } if (p[i] == '\r') { std::cout << "\\r"; } else if (p[i] == '\n') { std::cout << "\\n" << std::endl; debug_.newline = true; } else { std::cout << (isprint(p[i]) ? p[i] : '.'); } } if (length > 512) { std::cout << "..." << std::endl; debug_.newline = true; } } long sent = 0; const char* p = static_cast(buf); while (length > 0) { long len = 0; size_t retries = 0; const size_t maxTCPSocketRetries = 10 * 60; // 10 minutes errno = 0; len = ::write(socket_, p, length); while (len == 0) { Log::warning() << "Socket write returns zero (" << *this << ")" << Log::syserr << std::endl; if (++retries >= maxTCPSocketRetries) { Log::warning() << "Giving up." << std::endl; break; } Log::warning() << "Sleeping...." << std::endl; ::sleep(1); errno = 0; len = ::write(socket_, p, length); } if (len < 0) { Log::error() << "Socket write failed (" << *this << ")" << Log::syserr << std::endl; return len; } if (len == 0) { Log::warning() << "Socket write incomplete (" << *this << ") " << sent << " out of " << requested << std::endl; return sent; } sent += len; length -= len; p += len; } return sent; } long TCPSocket::read(void* buf, long length) const { if (length <= 0) { return length; } static bool useSelectOnTCPSocket = Resource("useSelectOnTCPSocket", false); long received = 0; char* p = static_cast(buf); bool nonews = false; while (length > 0) { long len; if (useSelectOnTCPSocket) { static long socketSelectTimeout = Resource("socketSelectTimeout", 0); Select select(socket_); bool more = socketSelectTimeout > 0; while (more) { more = false; if (!select.ready(socketSelectTimeout)) { SavedStatus save; Log::warning() << "No news from " << remoteHost() << " from " << Seconds(socketSelectTimeout) << std::endl; Log::status() << "No news from " << remoteHost() << " from " << Seconds(socketSelectTimeout) << std::endl; // FIXME: enable the nonews flag here? // nonews = true; // Time out, write 0 bytes to check that peer is alive if (::write(socket_, nullptr, 0) != 0) { Log::error() << "TCPSocket::read write" << Log::syserr << std::endl; return -1; } more = true; break; } } len = -1; if (nonews) { AutoAlarm alarm(60, true); Log::status() << "Resuming transfer" << std::endl; len = ::read(socket_, p, length); } else { len = ::read(socket_, p, length); } } else { len = ::read(socket_, p, length); } if (len < 0) { Log::error() << "Socket read failed (" << *this << ")" << Log::syserr << std::endl; return len; } if (len == 0) { return received; } if (debug_.on) { if (debug_.mode != 'r') { debug_.newline = true; std::cout << std::endl << std::endl; debug_.mode = 'r'; } for (long i = 0; i < std::min(len, 512L); i++) { if (debug_.newline) { std::cout << "<<< "; debug_.newline = false; } if (p[i] == '\r') { std::cout << "\\r"; } else if (p[i] == '\n') { std::cout << "\\n" << std::endl; debug_.newline = true; } else { std::cout << (isprint(p[i]) ? p[i] : '.'); } } if (len > 512) { std::cout << "..." << std::endl; debug_.newline = true; } } received += len; length -= len; p += len; } return received; } void TCPSocket::close() { if (socket_ != -1) { SYSCALL(::close(socket_)); } socket_ = -1; remotePort_ = localPort_ = -1; localHost_ = remoteHost_ = ""; localAddr_ = remoteAddr_ = none; } static jmp_buf env; static void catch_alarm(int) { longjmp(env, 1); } // This should be in the TCPClient.cc, but I want to reuse the Mutex // to lock any call to NIS with the same one TCPSocket& TCPClient::connect(const std::string& remote, int port, int retries, int timeout, int retryDelay) { std::string host = hostName(remote); in_addr_t addr; in_addr_t none = (in_addr_t)-1; hostent* him; sockaddr_in sin; ::memset(&sin, 0, sizeof(sin)); { // Block for local_mutex AutoLock lock(local_mutex); sin.sin_port = htons(port); sin.sin_family = AF_INET; addr = ::inet_addr(remote.c_str()); sin.sin_addr.s_addr = addr; if (addr == none) { // For some reason, sgi verion of gethostbyname_r is broken #if 0 hostent_data data; hostent host; // Due to a STUPID aix bug we need to clear zero(host); zero(data); if (gethostbyname_r(remote.c_str(), &host, &data)) him = 0; else him = &host; #else him = ::gethostbyname(remote.c_str()); #endif if (him == nullptr) { // Log::error() << "Unknown host [" << remote << "]" << std::endl; throw UnknownHost(remote); } sin.sin_family = him->h_addrtype; ::memcpy(&sin.sin_addr, him->h_addr_list[0], him->h_length); } } // End of local_mutex int tries = 0; int status = 0; do { int save_errno = 0; bind(); if (timeout) { if (setjmp(env) == 0) { void (*old)(int) = signal(SIGALRM, catch_alarm); alarm(timeout); status = ::connect(socket_, reinterpret_cast(&sin), sizeof(sin)); save_errno = errno; alarm(0); /// @todo change this to sigaction signal(SIGALRM, old); } else { throw TimeOut("connect", timeout); } } else { status = ::connect(socket_, reinterpret_cast(&sin), sizeof(sin)); save_errno = errno; } if (status < 0) { errno = save_errno; Log::error() << "connect to " << host << " " << port << Log::syserr << std::endl; Log::status() << "Connect: " << host << ":" << port << Log::syserr << " " << tries << '/' << retries << std::endl; ::close(socket_); socket_ = -1; errno = save_errno; switch (errno) { case ECONNREFUSED: if (++tries >= retries) { if (retries >= 0) { throw TooManyRetries(tries); } } ::sleep(retryDelay); break; case EINPROGRESS: // TODO: Potential file descriptor leak /* ::close(socket_); */ /* socket_ = -1; */ throw FailedSystemCall("connect"); #if 0 case ETIMEDOUT: Log::info() << "Waiting for network " << host << ":" << port << Log::syserr << std::endl; Log::status() << "Waiting for network " << host << ":" << port << Log::syserr << std::endl; break; #endif #if 0 case ENETUNREACH: case ETIMEDOUT: case ECONNRESET: case EADDRNOTAVAIL: case EAGAIN: #endif default: Log::status() << "Waiting for network " << host << ":" << port << Log::syserr << std::endl; #if 0 if (++tries >= retries) { if (retries != 0) throw TooManyRetries(tries); } #endif ::sleep(120); break; #if 0 default: throw FailedSystemCall("connect"); break; #endif } } } while (status < 0); remotePort_ = ntohs(sin.sin_port); remoteAddr_ = sin.sin_addr; remoteHost_ = addrToHost(sin.sin_addr); register_ignore_sigpipe(); return *this; } void set_socket_buffer_size(int& socket, const char* ssock, const int& stype, const int size) { Log::debug() << "Setting " << ssock << " buffer size " << size << std::endl; int flg = 0; socklen_t flgsize = sizeof(flg); if (getsockopt(socket, SOL_SOCKET, stype, &flg, &flgsize) < 0) { Log::warning() << "getsockopt " << ssock << " " << Log::syserr << std::endl; } if (flg != size) { if (setsockopt(socket, SOL_SOCKET, stype, &size, sizeof(size)) < 0) { Log::warning() << "setsockopt " << ssock << " " << Log::syserr << std::endl; } if (getsockopt(socket, SOL_SOCKET, stype, &flg, &flgsize) < 0) { Log::warning() << "getsockopt " << ssock << " " << Log::syserr << std::endl; } bool warn = (flg != size); #if defined(__linux__) // For Linux we ignore if the effective size is 2x what is set // see Linux 'man 7 socket' // when set using setsockopt() the Linux kernel doubles the socket buffer size // to allow space for bookkeeping overhead and this doubled value is // returned by getsockopt(). The minimum (doubled) value for this option is 2048. warn &= !(flg == 2 * size); #endif if (warn) { Log::warning() << "Attempt to set " << stype << " buffer size to " << size << " but kernel set size to " << flg << std::endl; } } } int TCPSocket::createSocket(int port, const SocketOptions& opts) { localPort_ = port; int s = ::socket(AF_INET, SOCK_STREAM, 0); if (s < 0) { throw FailedSystemCall("::socket"); } if (opts.reuseAddr()) { int flg = 1; if (::setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &flg, sizeof(flg)) < 0) { Log::warning() << "setsockopt SO_REUSEADDR" << Log::syserr << std::endl; } } if (opts.keepAlive()) { int flg = 1; if (::setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, &flg, sizeof(flg)) < 0) { Log::warning() << "setsockopt SO_KEEPALIVE" << Log::syserr << std::endl; } } if (opts.reusePort()) { #ifdef SO_REUSEPORT int flg = 1; SYSCALL(::setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &flg, sizeof(flg))); #endif } if (opts.noLinger()) { #ifdef SO_LINGER ///< turn off SO_LINGER linger ling; ling.l_onoff = 0; ling.l_linger = 0; if (::setsockopt(s, SOL_SOCKET, SO_LINGER, &ling, sizeof(ling)) < 0) { Log::warning() << "setsockopt SO_LINGER" << Log::syserr << std::endl; } #endif #ifdef SO_DONTLINGER if (::setsockopt(s, SOL_SOCKET, SO_DONTLINGER, NULL, 0) < 0) Log::warning() << "setsockopt SO_DONTLINGER" << Log::syserr << std::endl; #endif } if (opts.ipLowDelay()) { int tos = IPTOS_LOWDELAY; if (::setsockopt(s, IPPROTO_IP, IP_TOS, &tos, sizeof(tos)) < 0) { Log::warning() << "setsockopt IP_TOS" << Log::syserr << std::endl; } } if (opts.tcpNoDelay()) { int flag = 1; if (::setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char*)&flag, sizeof(flag)) < 0) { Log::warning() << "setsockopt TCP_NODELAY" << Log::syserr << std::endl; } } receiveBufferSize_ = receiveBufferSize_ ? receiveBufferSize_ : opts.receiveBufferSize(); if (receiveBufferSize_) { set_socket_buffer_size(s, "SO_RCVBUF", SO_RCVBUF, receiveBufferSize_); } sendBufferSize_ = sendBufferSize_ ? sendBufferSize_ : opts.sendBufferSize(); if (sendBufferSize_) { set_socket_buffer_size(s, "SO_SNDBUF", SO_SNDBUF, sendBufferSize_); } sockaddr_in sin; ::memset(&sin, 0, sizeof(struct sockaddr_in)); sin.sin_port = htons(localPort_); sin.sin_family = AF_INET; std::string addr = bindingAddress(); if (addr.length() == 0) { sin.sin_addr.s_addr = INADDR_ANY; } else { sin.sin_addr.s_addr = ::inet_addr(addr.c_str()); } while (::bind(s, reinterpret_cast(&sin), sizeof(sin)) == -1) { Log::warning() << "bind port " << localPort_ << " " << addr << Log::syserr << std::endl; ::sleep(5); } AutoLock lock(local_mutex); #ifdef SGI int len = sizeof(sin); #else socklen_t len = sizeof(sin); #endif ::getsockname(s, reinterpret_cast(&sin), &len); if (localPort_ != 0) { int gotPort = ntohs(sin.sin_port); if (localPort_ != gotPort) { std::ostringstream msg; msg << "TCPSocket::newSocket() asking for port " << localPort_ << " but got " << gotPort << std::endl; throw eckit::SeriousBug(msg.str(), Here()); } } localPort_ = ntohs(sin.sin_port); localAddr_ = sin.sin_addr; localHost_ = addrToHost(sin.sin_addr); if (localHost_ == "0.0.0.0") { if (addr.length() == 0) { AutoLock lock(local_mutex); localHost_ = Resource("host", ""); if (localHost_.length() == 0) { localHost_ = eckit::Main::hostname(); } } else { localHost_ = addr; } } // Set the socket 'close on exec' SYSCALL(fcntl(s, F_SETFD, FD_CLOEXEC)); return s; } void TCPSocket::bind() {} static std::map cache; std::string TCPSocket::addrToHost(in_addr addr) { AutoLock lock(local_mutex); // For some reason IBM's gethostbyaddr_r dumps core // from time to time, so let's cache the result // to minimise the cores std::map::iterator j = cache.find(addr.s_addr); if (j != cache.end()) { return (*j).second; } hostent* h; // For some reason, sgi verion of gethostbyname_r is broken #ifdef _AIX hostent_data data; hostent host; // Due to a STUPID aix bug we need to clear zero(host); zero(data); if (gethostbyaddr_r(reinterpret_cast(&addr), sizeof(addr), AF_INET, &host, &data)) h = 0; else h = &host; #else h = gethostbyaddr(reinterpret_cast(&addr), sizeof(addr), AF_INET); #endif std::string s = h ? std::string(h->h_name) : IPAddress(addr).asString(); cache[addr.s_addr] = s; return s; } std::string TCPSocket::hostName(const std::string& h, bool full) { in_addr_t addr = ::inet_addr(h.c_str()); if (addr == (in_addr_t)-1) { if (full) { return h; } return h.substr(0, h.find('.')); } struct in_addr a; a.s_addr = addr; std::string s = addrToHost(a); if (!full && !isdigit(s[0])) { return s.substr(0, s.find('.')); } return s; } void TCPSocket::register_ignore_sigpipe() { #if 0 ::signal(SIGPIPE, SIG_IGN); #else struct sigaction act; eckit::zero(act); act.sa_handler = SIG_IGN; act.sa_flags = SA_RESTART; SYSCALL(::sigaction(SIGPIPE, &act, nullptr)); //< shouldn't fail -- see ERROR conditions in man(2) sigaction #endif } int TCPSocket::socket() { return socket_; } in_addr TCPSocket::remoteAddr() const { return remoteAddr_; } const std::string& TCPSocket::remoteHost() const { return remoteHost_; } int TCPSocket::remotePort() const { return remotePort_; } in_addr TCPSocket::localAddr() const { ((net::TCPSocket*)this)->bind(); return localAddr_; } const std::string& TCPSocket::localHost() const { ((net::TCPSocket*)this)->bind(); return localHost_; } int TCPSocket::localPort() const { ((net::TCPSocket*)this)->bind(); return localPort_; } std::string TCPSocket::bindingAddress() const { return ""; } long TCPSocket::rawRead(void* buf, long length) { return ::read(socket_, buf, length); } bool TCPSocket::stillConnected() const { if (socket_ == -1) { return false; } fd_set r; fd_set e; fd_set w; FD_ZERO(&r); FD_SET(socket_, &r); FD_ZERO(&e); FD_SET(socket_, &e); FD_ZERO(&w); FD_SET(socket_, &w); ::timeval tv = {0, 0}; if (::select(socket_ + 1, &r, &w, &e, &tv) >= 0) { if (!FD_ISSET(socket_, &r)) { return true; } int n = 0; if (::ioctl(socket_, FIONREAD, &n) < 0) { Log::info() << "TCPSocket::stillConnected(FIONREAD) failed " << Log::syserr << std::endl; return false; } if (n == 0) { Log::warning() << "TCPSocket::stillConnected => connection lost" << std::endl; return false; } return true; } Log::info() << "TCPSocket::stillConnected(select) failed " << Log::syserr << std::endl; return false; } void TCPSocket::debug(bool on) { debug_.on = on; debug_.newline = true; debug_.mode = 0; } void TCPSocket::print(std::ostream& s) const { s << "TCPSocket[fd=" << socket_ << ",remote[" << remoteHost() << ":" << remotePort() << " (" << remoteAddr_ << ")]" << ",local[" << localHost() << ":" << localPort() << " (" << localAddr_ << ")]" // << ",localPort=" << localPort_ // << ",remotePort=" << remotePort_ // << ",remoteHost=" << remoteHost_ // << ",remoteAddr=" << remoteAddr_ // << ",localHost=" << localHost_ // << ",localAddr_=" << localAddr_ << "]"; } std::ostream& operator<<(std::ostream& s, in_addr a) { s << IPAddress(a); return s; } } // namespace eckit::net eckit-2.0.7/src/eckit/net/TCPServer.h0000664000175000017500000000421615161702250017447 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Jun 96 #ifndef eckit_TCPServer_h #define eckit_TCPServer_h #include "eckit/net/TCPSocket.h" #include "eckit/thread/Mutex.h" namespace eckit::net { class TCPServer : public TCPSocket { public: TCPServer(const SocketOptions& = SocketOptions::server()); explicit TCPServer(int port, const SocketOptions& = SocketOptions::server()); TCPServer(const TCPServer&) = delete; TCPServer& operator=(const TCPServer&) = delete; TCPServer(TCPServer&&) = delete; TCPServer& operator=(TCPServer&&) = delete; ~TCPServer() override; void willFork(bool); // accept a client, more can be accepted virtual TCPSocket& accept(const std::string& message = "Waiting for connection", int timeout = 0, bool* connected = nullptr); void closeExec(bool on) { closeExec_ = on; } int socket() override; void close() override; protected: // members int port_; int listen_; SocketOptions options_; //< options to build the socket protected: // methods void bind() override; void print(std::ostream& s) const override; private: // methods // To be used by Select std::string bindingAddress() const override; private: // members bool closeExec_; Mutex mutex_; }; //---------------------------------------------------------------------------------------------------------------------- class EphemeralTCPServer : public TCPServer { public: EphemeralTCPServer(const SocketOptions& = SocketOptions::data()); explicit EphemeralTCPServer(int port, const SocketOptions& = SocketOptions::data()); ~EphemeralTCPServer() override = default; }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/Telnetable.h0000664000175000017500000000163015161702250017706 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Telnetable.h // Baudouin Raoult - ECMWF Jun 96 #ifndef eckit_Telnetable_h #define eckit_Telnetable_h #include "eckit/thread/ThreadControler.h" namespace eckit::net { /// A telnet-able object class Telnetable { public: // -- Contructors Telnetable(int port); // -- Destructor ~Telnetable(); private: // No copy allowed Telnetable(const Telnetable&); Telnetable& operator=(const Telnetable&); // -- Members ThreadControler telnet_; }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/UDPClient.cc0000664000175000017500000000534515161702250017563 0ustar alastairalastair/* * (C) Copyright 1996-2017 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include #include #include #include #include #include "eckit/config/Configuration.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Bytes.h" #include "eckit/net/UDPClient.h" #include "eckit/utils/Translator.h" namespace eckit::net { UDPClient::UDPClient(const Configuration& cfg) : hostname_(cfg.getString("host")), port_(cfg.getInt("port")), socketfd_(0), servinfo_{nullptr}, addr_{nullptr} { init(); } UDPClient::UDPClient(const std::string& hostname, int port) : hostname_(hostname), port_(port), socketfd_(0), servinfo_{nullptr}, addr_{nullptr} { init(); } void UDPClient::init() { struct addrinfo hints; ::memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_DGRAM; Translator toStr; int err = 0; if ((err = ::getaddrinfo(hostname_.c_str(), toStr(port_).c_str(), &hints, &servinfo_)) != 0) { std::ostringstream msg; msg << "getaddrinfo failed in UDPClient with " << " hostname=" << hostname_ << " port=" << port_ << " -- " << ::gai_strerror(err); throw FailedSystemCall(msg.str()); } // loop through all the addrinfo results and make a socket for (addr_ = servinfo_; addr_ != nullptr; addr_ = addr_->ai_next) { if ((socketfd_ = ::socket(addr_->ai_family, addr_->ai_socktype, addr_->ai_protocol)) == -1) { continue; } break; } if (addr_ == nullptr) { std::ostringstream msg; msg << "UDPClient failed to create a socket"; throw FailedSystemCall(msg.str()); } } UDPClient::~UDPClient() { ::freeaddrinfo(servinfo_); SYSCALL(::close(socketfd_)); } void UDPClient::send(const void* buffer, size_t length) { ssize_t sent = ::sendto(socketfd_, buffer, length, 0, addr_->ai_addr, addr_->ai_addrlen); if (sent == -1) { std::ostringstream msg; msg << "UDPClient failed to send " << Bytes(length) << " to host " << hostname_; throw FailedSystemCall(msg.str()); } } void UDPClient::print(std::ostream& s) const { s << "UDPClient[hostname=" << hostname_ << ",port=" << port_ << ",socketfd=" << socketfd_ << "]"; } } // namespace eckit::net eckit-2.0.7/src/eckit/net/TelnetUser.h0000664000175000017500000000147315161702250017726 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_net_TelnetUser_h #define eckit_net_TelnetUser_h #include "eckit/net/NetUser.h" namespace eckit::net { class TelnetUser : public NetUser { public: explicit TelnetUser(net::TCPSocket&); ~TelnetUser() override; static void terminate(TelnetUser& other) { other.stop(); } private: void serve(Stream&, std::istream&, std::ostream&) override; std::string from_; }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/NetService.cc0000664000175000017500000001010215161702250020026 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/NetService.h" #include "eckit/config/Resource.h" #include "eckit/io/Select.h" #include "eckit/log/Log.h" #include "eckit/net/NetUser.h" #include "eckit/runtime/Monitor.h" #include "eckit/runtime/ProcessControler.h" #include "eckit/thread/ThreadControler.h" namespace eckit::net { class NetServiceProcessControler : public ProcessControler { public: NetServiceProcessControler(const std::string& name, NetUser* user, TCPServer& server, long parent, bool visible); private: std::string name_; std::unique_ptr user_; TCPServer& server_; long parent_; bool visible_; virtual void run(); virtual void afterForkInParent(); virtual void afterForkInChild(); }; NetService::NetService(int port, bool visible, const SocketOptions& options) : server_(port, options), visible_(visible) {} NetService::~NetService() {} std::string NetService::hostname() const { return server_.localHost(); } int NetService::port() const { return server_.localPort(); } void NetService::run() { Monitor::instance().show(visible_); Monitor::instance().name(name()); Monitor::instance().kind(name()); std::ostringstream oss; oss << "Waiting on port " << port(); while (!stopped()) { Log::status() << oss.str() << std::endl; if (timeout()) { Select select(server_); if (!select.ready(timeout())) { // This will allow to check stopped() again continue; } } NetUser* user = newUser(server_.accept(oss.str())); if (runAsProcess()) { NetServiceProcessControler t(name(), user, server_, Monitor::instance().self(), visible_); t.start(); } else { ThreadControler t(user); t.start(); } } } bool NetService::runAsProcess() const { return Resource(name() + "NetServiceForkProcess", preferToRunAsProcess()); } bool NetService::preferToRunAsProcess() const { return false; } long NetService::timeout() const { return 0; } //---------------------------------------------------------------------------------------------------------------------- NetServiceProcessControler::NetServiceProcessControler(const std::string& name, NetUser* user, TCPServer& server, long parent, bool visible) : ProcessControler(true), name_(name), user_(user), server_(server), parent_(parent), visible_(visible) { Log::info() << "NetServiceProcessControler::NetServiceProcessControler" << std::endl; } void NetServiceProcessControler::run() { eckit::Monitor::instance().reset(); // needed to the monitor to work on forked (but not execed process) Monitor::instance().parent(parent_); Monitor::instance().name(name_); // Monitor::instance().kind(name_); Monitor::instance().show(visible_); Log::info() << "NetServiceProcessControler::run start" << std::endl; try { user_->run(); } catch (std::exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; Log::error() << "** Exception is ignored" << std::endl; } Log::info() << "NetServiceProcessControler::run end" << std::endl; } void NetServiceProcessControler::afterForkInParent() { // This will close the connected socket Log::info() << "NetServiceProcessControler::run afterForkInParent" << std::endl; user_.reset(nullptr); } void NetServiceProcessControler::afterForkInChild() { // Close the accept() socket that is used in the parent Log::info() << "NetServiceProcessControler::run afterForkInChild" << std::endl; server_.close(); } } // namespace eckit::net eckit-2.0.7/src/eckit/net/ProxiedTCPServer.h0000664000175000017500000000205715161702250021003 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Aug 2017 #ifndef eckit_ProxiedTCPServer_h #define eckit_ProxiedTCPServer_h #include "eckit/net/TCPServer.h" namespace eckit { class ProxiedTCPServer : public net::TCPServer { public: ProxiedTCPServer(int port = 0, const net::SocketOptions& = net::SocketOptions::server()); ~ProxiedTCPServer() override; // accept a client, more can be accepted virtual net::TCPSocket& accept(const std::string& message = "Waiting for connection", int timeout = 0, bool* connected = nullptr) override; private: void print(std::ostream& s) const override; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/net/Telnet.cc0000664000175000017500000000132015161702250017214 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/Telnet.h" #include "eckit/net/TelnetUser.h" #include "eckit/runtime/Monitor.h" namespace eckit::net { Telnet::Telnet(int port) : NetService(port) {} Telnet::~Telnet() {} net::NetUser* Telnet::newUser(net::TCPSocket& protocol) const { return new TelnetUser(protocol); } } // namespace eckit::net eckit-2.0.7/src/eckit/net/Connector.h0000664000175000017500000000716015161702250017565 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Jun 2011 #ifndef eckit_net_Connector_h #define eckit_net_Connector_h #include "eckit/io/BufferCache.h" #include "eckit/io/Length.h" #include "eckit/io/cluster/NodeInfo.h" #include "eckit/net/TCPSocket.h" #include "eckit/serialisation/Stream.h" namespace eckit::net { //---------------------------------------------------------------------------------------------------------------------- class ConnectorException : public Exception { virtual bool retryOnServer() const { return true; } virtual bool retryOnClient() const { return true; } public: ConnectorException(const std::string& what) : Exception(what) {} }; //---------------------------------------------------------------------------------------------------------------------- class Connector : public Stream { public: Connector(const std::string& name, const std::string& node); ~Connector(); // -- Methods void lock(); void unlock(); void reset(); void check(); bool locked() const { return locked_; } const std::string& host() const { return host_; } const std::string& node() const { return node_; } void autoclose(bool on) { autoclose_ = on; } void memoize(bool on, unsigned long time); static Connector& service(const NodeInfo& node); static Connector& service(const std::string& name, const std::string& node); static NodeInfo nodeInfo(const std::string& name, const std::string& node); protected: // -- Members // None Connector(const std::string&, int, const std::string&); // -- Methods void print(std::ostream&) const override; // -- Class members // None static Connector& get(const std::string& host, int port, const std::string&); // -- Class methods // None private: // -- Members std::string host_; std::string node_; int port_; TCPSocket socket_; bool locked_; time_t last_; // Memoisation bool memoize_; bool sent_; unsigned long life_; BufferCache out_; BufferCache in_; bool autoclose_; std::map cache_; struct { const char* buffer_; size_t pos_; size_t size_; } cached_; // -- Methods // None TCPSocket& socket(); template long socketIo(F proc, T buf, long len, const char*, time_t&); // -- Overridden methods // None // From Stream long write(const void* buf, long len) override; long read(void* buf, long len) override; std::string name() const override; // -- Friends friend std::ostream& operator<<(std::ostream& s, const Connector& p) { p.print(s); return s; } friend class ConnectorCache; }; //---------------------------------------------------------------------------------------------------------------------- class AutoMemoize { Connector& c_; unsigned long t_; public: AutoMemoize(Connector& c, unsigned long t) : c_(c), t_(t) { c_.memoize(true, t_); } ~AutoMemoize() { c_.memoize(false, t_); } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/Port.h0000664000175000017500000000133015161702250016550 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Jul 96 #ifndef eckit_net_Port_h #define eckit_net_Port_h #include namespace eckit::net { class Port { public: Port(const std::string&, int); operator int() const { return port_; } private: int port_; }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/TCPStream.h0000664000175000017500000000501215161702250017427 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File TCPStream.h // Baudouin Raoult - ECMWF May 96 #ifndef eckit_TCPStream_h #define eckit_TCPStream_h #include "eckit/memory/Counted.h" #include "eckit/net/TCPSocket.h" #include "eckit/serialisation/Stream.h" namespace eckit::net { class TCPServer; //---------------------------------------------------------------------------------------------------------------------- class TCPStreamBase : public Stream { public: TCPStreamBase() {} in_addr remoteAddr() { return socket().remoteAddr(); } long write(const void* buf, long len) override { return socket().write(buf, len); } long read(void* buf, long len) override { return socket().read(buf, len); } protected: std::string name() const override; private: // methods std::string nonConstName(); virtual TCPSocket& socket() = 0; }; //---------------------------------------------------------------------------------------------------------------------- class TCPStream : public TCPStreamBase { public: /// @note Takes ownership of TCPSocket; TCPStream(net::TCPSocket&); ~TCPStream() override; TCPSocket& socket() override { return socket_; } protected: // members TCPSocket socket_; private: TCPStream(TCPServer&); void closeOutput() override; }; //---------------------------------------------------------------------------------------------------------------------- class InstantTCPStream : public TCPStreamBase { public: /// @note does not take ownership of TCPSocket InstantTCPStream(net::TCPSocket& socket) : socket_(socket) {} TCPSocket& socket() override { return socket_; } private: InstantTCPStream(TCPServer&); TCPSocket& socket_; }; //---------------------------------------------------------------------------------------------------------------------- class SharedTCPStream : public TCPStream, public Counted { public: SharedTCPStream(net::TCPSocket&); private: ~SharedTCPStream() override; SharedTCPStream(TCPServer&); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/UDPClient.h0000664000175000017500000000260015161702250017414 0ustar alastairalastair/* * (C) Copyright 1996-2017 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_net_UDPClient_h #define eckit_net_UDPClient_h #include #include namespace eckit { class Buffer; class Configuration; namespace net { class UDPClient { public: // methods explicit UDPClient(const Configuration& cfg); UDPClient(const std::string& hostname, int port); UDPClient(const UDPClient&) = delete; UDPClient& operator=(const UDPClient&) = delete; UDPClient(UDPClient&&) = delete; UDPClient& operator=(UDPClient&&) = delete; ~UDPClient(); void send(const void* buf, size_t length); protected: // methods void print(std::ostream& s) const; friend std::ostream& operator<<(std::ostream& s, const UDPClient& socket) { socket.print(s); return s; } private: // members std::string hostname_; int port_; int socketfd_; struct addrinfo* servinfo_; struct addrinfo* addr_; void init(); }; } // namespace net } // namespace eckit #endif eckit-2.0.7/src/eckit/net/Connector.cc0000664000175000017500000002752215161702250017727 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/Connector.h" #include "eckit/config/Resource.h" #include "eckit/io/cluster/ClusterNodes.h" #include "eckit/log/Seconds.h" #include "eckit/net/TCPClient.h" #include "eckit/net/TCPStream.h" #include "eckit/thread/ThreadSingleton.h" namespace eckit::net { //---------------------------------------------------------------------------------------------------------------------- static void offLine(const std::string& host, int port) { static bool setNodeOfflineOnError = Resource("setNodeOfflineOnError", false); if (setNodeOfflineOnError) { ClusterNodes::offLine(host, port); } } Connector::Connector(const std::string& host, int port, const std::string& node) : host_(host), node_(node), port_(port), locked_(false), last_(::time(nullptr)), memoize_(false), sent_(false), life_(0), autoclose_(false) { Log::info() << "Connector::Connector(" << node << "," << host << ":" << port << ")" << std::endl; } Connector::~Connector() { socket_.close(); // try { // if (socket_.isConnected()) { // (*this) << "bye"; // } // } // catch (std::exception& e) { // Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; // Log::error() << "** Exception is ignored" << std::endl; // } } TCPSocket& Connector::socket() { static int connectorTimeout = Resource("connectorTimeout", 0); if (connectorTimeout != 0) { time_t now = ::time(nullptr); if (now - last_ > connectorTimeout) { Log::info() << "Connector::socket() opened for " << Seconds(now - last_) << " seconds, reopening connection" << std::endl; socket_.close(); } } if (!socket_.isConnected()) { try { NodeInfo remote; TCPClient client(SocketOptions::control()); Log::info() << "Connector::stream connecting to " << host_ << ":" << port_ << std::endl; socket_ = client.connect(host_, port_, -1); InstantTCPStream s(socket_); // Login remote = NodeInfo::sendLogin(s); ClusterNodes::onLine(host_, port_); } catch (std::exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; Log::error() << "** Exception is handled" << std::endl; offLine(host_, port_); std::ostringstream os; os << name() << ": " << e.what(); throw ConnectorException(os.str()); } } return socket_; } void Connector::check() { if (socket_.isConnected()) { try { if (!socket_.stillConnected()) { socket_.close(); offLine(host_, port_); /// @todo maybe remove this from here, substitute with a descriptive exception } } catch (std::exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; Log::error() << "** Exception is handled" << std::endl; socket_.close(); offLine(host_, port_); /// @todo maybe remove this from here } } } void Connector::print(std::ostream& os) const { os << "Connector[host=" << host_ << ",port=" << port_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- class ConnectorCache { using Cache = std::multimap, Connector*>; Cache cache_; public: /// @note Lazy construction of singleton. /// This is required to ensure correct and portable order of initialisation and destruction /// across multiple architectures. /// Do not remove this function or change it to a static variable in this translation unit. static ConnectorCache& instance() { static ThreadSingleton cache; return cache.instance(); } Connector& find(const std::string& host, int port, const std::string& node) { std::pair p(host, port); std::pair r = cache_.equal_range(p); for (Cache::iterator j = r.first; j != r.second; ++j) { if (!((*j).second)->locked()) { (*j).second->check(); return *(*j).second; } } Connector* c = new Connector(host, port, node); cache_.insert(make_pair(p, c)); return *c; } ~ConnectorCache() { for (Cache::iterator j = cache_.begin(); j != cache_.end(); ++j) { Connector* c = (*j).second; delete c; } } void reset() { for (Cache::iterator j = cache_.begin(); j != cache_.end(); ++j) { Connector* c = (*j).second; c->reset(); } } }; class NodeInfoCache { using Cache = std::map, NodeInfo>; Cache cache_; public: /// @note Lazy construction of singleton. /// This is required to ensure correct and portable order of initialisation and destruction /// across multiple architectures. /// Do not remove this function or change it to a static variable in this translation unit. static NodeInfoCache& instance() { static ThreadSingleton cache; return cache.instance(); } NodeInfo& find(Stream& s, const std::string& name, const std::string& node) { std::pair p(name, node); Cache::iterator j = cache_.find(p); if (j != cache_.end()) { return (*j).second; } // Log::info() << "Connector::nodeInfo(" << name << "," << node << ")" << std::endl; s << "info"; s << name; s << node; NodeInfo info; bool ok; s >> ok; if (!ok) { std::ostringstream os; os << "Cannot get node info for " << name << "@" << node; throw ConnectorException(os.str()); } s >> cache_[p]; return cache_[p]; } ~NodeInfoCache() {} void reset() { cache_.clear(); } }; //---------------------------------------------------------------------------------------------------------------------- Connector& Connector::get(const std::string& host, int port, const std::string& name) { // Log::info() << "Connector::get(" << host << "," << port << ")" << std::endl; return ConnectorCache::instance().find(host, port, name); } Connector& Connector::service(const std::string& name, const std::string& node) { // Log::info() << "Connector::service(" << name << "," << node << ")" << std::endl; NodeInfo info = ClusterNodes::lookUp(name, node); return get(info.host(), info.port(), info.node()); } Connector& Connector::service(const NodeInfo& info) { return get(info.host(), info.port(), info.node()); } void Connector::lock() { ASSERT(!locked_); locked_ = true; } void Connector::unlock() { ASSERT(locked_); locked_ = false; if (autoclose_) { reset(); } } void Connector::reset() { in_.reset(); out_.reset(); cache_.clear(); try { socket_.close(); } catch (std::exception& e) { Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; Log::error() << "** Exception is ignored" << std::endl; } } std::string Connector::name() const { std::ostringstream os; os << "Connector[" << node_ << "," << host_ << ":" << port_ << "]"; return os.str(); } template long Connector::socketIo(F proc, T buf, long len, const char* msg, time_t& last) { TCPSocket& s = socket(); last = ::time(nullptr); long l = (s.*proc)(buf, len); if (l != len) { reset(); ConnectorCache::instance().reset(); NodeInfoCache::instance().reset(); std::ostringstream os; os << "Connector::socketIo(" << name() << ") only " << l << " byte(s) " << msg << " intead of " << len << Log::syserr; // throw ConnectorException(std::string(os)); throw Retry(os.str()); } return l; } long Connector::write(const void* buf, long len) { if (in_.count()) { in_.reset(); out_.count(); } if (memoize_) { sent_ = false; out_.add(buf, len); return len; } return socketIo(&TCPSocket::write, buf, len, "written", last_); } long Connector::read(void* buf, long len) { if (memoize_) { if (!sent_) { std::map::iterator j = cache_.find(out_); bool useCache = false; if (j != cache_.end()) { // cout << "MEMOIZE IN CACHE " << (*j).first << std::endl; if ((::time(nullptr) - (*j).second.updated()) > long(life_)) { // cout << " CACHE IS STALE" << (*j).first << std::endl; } else { useCache = true; cached_.buffer_ = (const char*)(*j).second.buffer(); cached_.size_ = (*j).second.count(); cached_.pos_ = 0; sent_ = true; } } if (!useCache) { cached_.buffer_ = nullptr; try { ASSERT((size_t)socketIo(&TCPSocket::write, out_.buffer(), out_.count(), "written", last_) == out_.count()); } catch (...) { reset(); throw; } sent_ = true; } } if (cached_.buffer_) { long left = cached_.size_ - cached_.pos_; long l = left < len ? left : len; if (l != len) { std::ostringstream os; os << "Connector::socketIo(" << name() << ") only " << l << " byte(s) memoized intead of " << len << Log::syserr; reset(); throw ConnectorException(os.str()); } ::memcpy(buf, cached_.buffer_ + cached_.pos_, len); cached_.pos_ += len; return len; } } try { len = socketIo(&TCPSocket::read, buf, len, "read", last_); } catch (...) { reset(); throw; } if (memoize_) { ASSERT(len > 0); in_.add(buf, len); } return len; } void Connector::memoize(bool on, unsigned long life) { ASSERT(on != memoize_); memoize_ = on; life_ = life; if (on) { ASSERT(in_.count() == 0); ASSERT(out_.count() == 0); sent_ = false; cached_.buffer_ = nullptr; if (cache_.size() > 10000) { // Log::info() << "Clear memoize cache" << std::endl; cache_.clear(); } } else { // cout << "Connector::memoize " << in_.count() << " " << out_.count() << std::endl; // cout << "-> " << out_ << std::endl; if (cached_.buffer_) { // cout << " CACHED" << std::endl; // cout << " .... " << cache_[out_] << std::endl; } else { // cout << "<- " << in_ << std::endl; cache_[out_] = in_; } in_.reset(); out_.reset(); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::net eckit-2.0.7/src/eckit/net/ProxiedTCPClient.h0000664000175000017500000000212515161702250020747 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Aug 2017 #ifndef eckit_ProxiedTCPClient_h #define eckit_ProxiedTCPClient_h #include "eckit/net/Endpoint.h" #include "eckit/net/TCPClient.h" namespace eckit { class ProxiedTCPClient : public net::TCPClient { public: ProxiedTCPClient(const std::string& proxyHost, int proxyPort, int port = 0); ~ProxiedTCPClient() override; using net::TCPClient::connect; net::TCPSocket& connect(const std::string& host, int port, int retries = 5, int timeout = 0, int retryDelay = 5) override; private: net::Endpoint proxy_; void print(std::ostream& s) const override; }; } // namespace eckit #endif eckit-2.0.7/src/eckit/net/HttpHeader.cc0000664000175000017500000002072615161702250020024 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/HttpHeader.h" #include "eckit/log/Log.h" #include "eckit/net/TCPSocket.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- // Check the HTTP/1.0 syntax // Check the HTTP/1.1 syntax const std::string WWW_Authenticate = "WWW-Authenticate"; const std::string Authorization = "Authorization"; const std::string Content_Type = "Content-Type"; const std::string Content_Length = "Content-Length"; const std::string Location = "Location"; const std::string DefaultType = "application/x-www-form-urlencoded"; const std::string Retry_After = "Retry-After"; bool HttpHeader::compare::operator()(const std::string& a, const std::string& b) const { return strcasecmp(a.c_str(), b.c_str()) < 0; } HttpHeader::HttpHeader() : version_("HTTP/1.0"), statusCode_(HttpError::OK), contentLength_(0), received_(false) { header_[Content_Type] = "text/html"; header_["Cache-Control"] = "no-cache"; header_["MIME-Version"] = "1.0"; } HttpHeader& HttpHeader::operator=(std::map >& parsed) { for (std::map >::const_iterator i = parsed.begin(); i != parsed.end(); ++i) { header_[(*i).first] = (*i).second; } Map::const_iterator j = header_.find(Content_Length); if (j != header_.end()) { contentLength_ = atol(((*j).second).c_str()); } else { contentLength_ = 0; } return *this; } HttpHeader::~HttpHeader() {} void HttpHeader::print(std::ostream& s) const { const char* CRLF = "\r\n"; // Status line + CRLF s << version_ << ' ' << statusCode_ << ' '; if (message_.empty()) { switch (statusCode_) { case HttpError::OK: s << "OK"; break; case HttpError::CREATED: s << "Created"; break; case HttpError::ACCEPTED: s << "Accepted"; break; case HttpError::NO_CONTENT: s << "No Content"; break; case HttpError::SEE_OTHER: s << "See Other"; break; case HttpError::NOT_FOUND: s << "Not Found"; break; case HttpError::NOT_IMPLEMENTED: s << "Not Implemented"; break; case HttpError::INTERNAL_SERVER_ERROR: s << "Internal Server Error"; break; case HttpError::BAD_REQUEST: s << "Bad Request"; break; case HttpError::UNAUTHORIZED: s << "Unauthorized"; break; } } else { s << message_; } s << CRLF; for (Map::const_iterator i = header_.begin(); i != header_.end(); ++i) { s << (*i).first << ": " << (*i).second << CRLF; } if (!received_) { s << Content_Length << ": " << contentLength_ + content_.size() << CRLF; } s << CRLF; long len = content_.size(); const char* p = static_cast(content_.data()); while (len-- > 0) { s.put(*p++); } } void HttpHeader::forward(const std::string& s) { header_[Location] = s; } void HttpHeader::retryAfter(long s) { std::ostringstream oss; oss << s; header_[Retry_After] = oss.str(); Log::debug() << "HttpHeader::retryAfter " << s << std::endl; Log::debug() << *this << std::endl; } void HttpHeader::length(const long l) { contentLength_ = l; } long HttpHeader::contentLength() const { return contentLength_; } void HttpHeader::type(const std::string& s) { header_[Content_Type] = s; } const std::string& HttpHeader::type() const { Map::const_iterator i = header_.find(Content_Type); if (i != header_.end()) { return (*i).second; } return DefaultType; } void HttpHeader::status(const long code, const std::string& message) { statusCode_ = code; message_ = message; } void HttpHeader::authenticate(const std::string& login) { header_[WWW_Authenticate] = ("Basic realm=\"" + login + "\""); status(HttpError::UNAUTHORIZED); } void HttpHeader::dontCache() {} bool HttpHeader::authenticated() const { Map::const_iterator i = header_.find(Authorization); if (i != header_.end()) { const char* s = (*i).second.c_str(); while (*s != ' ' && *s != '\t') { s++; } while (*s == ' ' || *s == '\t') { s++; } unsigned char b64[256]; for (int j = 0; j < 256; j++) { b64[j] = 64; } for (unsigned char c = 'A'; c <= 'Z'; c++) { b64[c] = c - 'A'; } for (unsigned char c = 'a'; c <= 'z'; c++) { b64[c] = c - 'a' + 26; } for (unsigned char c = '0'; c <= '9'; c++) { b64[c] = c - '0' + 52; } b64[int('+')] = 62; b64[int('/')] = 63; const unsigned char* p = (const unsigned char*)s; std::string q; int n = 2; while (b64[*p] < 64 && b64[*(p + 1)] < 64) { q += char((b64[p[0]] << n) | (b64[p[1]] >> (6 - n))); n += 2; if (n == 8) { p++; n = 2; } p++; } std::cout << q << std::endl; Tokenizer parse(":"); std::vector v; parse(q, v); if (v.size() == 2 && v[0] == "mars" && v[1] == "clave") { Log::info() << "client authenticated " << q << " -> " << (*i).second << std::endl; return true; } Log::info() << "client denied " << q << " -> " << (*i).second << std::endl; } return false; } void HttpHeader::content(const char* p, long len) { content_.openForWrite(0); content_.write(p, len); content_.close(); } void HttpHeader::setHeader(const std::string& k, const std::string& v) { header_[k] = v; } const std::string& HttpHeader::getHeader(const std::string& k) const { return ((HttpHeader*)this)->header_[k]; } std::string HttpHeader::content() const { return content_.str(); } //---------------------------------------------------------------------------------------------------------------------- static std::string nextLine(net::TCPSocket& socket) { char c; std::string s; for (;;) { ASSERT(socket.read(&c, 1) == 1); if (c == '\r') { ASSERT(socket.read(&c, 1) == 1); ASSERT(c == '\n'); return s; } s += c; ASSERT(s.length() < 32768); } } HttpHeader::HttpHeader(net::TCPSocket& socket) : received_(true) { std::string line = nextLine(socket); size_t i = line.find_first_of(' '); ASSERT(i != std::string::npos); version_ = line.substr(0, i); line.erase(0, i + 1); i = line.find_first_of(' '); ASSERT(i != std::string::npos); statusCode_ = Translator()(line.substr(0, i)); line.erase(0, i + 1); message_ = line; line = nextLine(socket); while (!line.empty()) { size_t i = line.find_first_of(':'); ASSERT(i != std::string::npos); std::string key = line.substr(0, i); while (!key.empty() && key[key.length() - 1] == ' ') { key.erase(key.length() - 1, 1); } std::string value = line.substr(i + 1, line.length()); while (!value.empty() && value[0] == ' ') { value.erase(0, 1); } header_[key] = value; line = nextLine(socket); } } void HttpHeader::checkForStatus() const { if (statusCode_ != HttpError::OK) { throw HttpError(statusCode_, message_); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/net/IPAddress.h0000664000175000017500000000236315161702250017451 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Dec 2019 #ifndef eckit_net_IPAddress_H #define eckit_net_IPAddress_H #include #include #include namespace eckit::net { class IPAddress { public: // Contructors IPAddress(const in_addr& address) : address_(address) {} IPAddress(const std::string&); IPAddress(const char*); std::string asString() const; const in_addr& address() const { return address_; } static IPAddress myIPAddress(); static IPAddress hostAddress(const std::string& hostname); bool operator==(const IPAddress& other) const; private: // Members in_addr address_; // Methods void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const IPAddress& p) { p.print(s); return s; } }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/SocketOptions.cc0000664000175000017500000000556315161702250020602 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/config/Resource.h" #include "eckit/net/SocketOptions.h" namespace eckit::net { static void init(SocketOptions& opts) { static std::string bindAddr = Resource("localBindingAddress", ""); /* "127.0.0.1" */ static int ListenBacklog = eckit::Resource("socketOptionsListenBacklog", SOMAXCONN); static bool reusePort = eckit::Resource("socketOptionsReusePort", false); static bool reuseAddr = eckit::Resource("socketOptionsReuseAddr", false); static bool noLinger = eckit::Resource("socketOptionsNoLinger", false); static bool keepAlive = eckit::Resource("socketOptionsKeepAlive", true); static bool ipLowDelay = eckit::Resource("socketOptionsIpLowDelay", true); static bool tcpNoDelay = eckit::Resource("socketOptionsTcpNoDelay", true); static int receiveBufferSize = eckit::Resource("socketOptionsReceiveBufferSize", 0); static int sendBufferSize = eckit::Resource("socketOptionsSendBufferSize", 0); opts.bindAddress(bindAddr); opts.listenBacklog(ListenBacklog); opts.reusePort(reusePort); opts.reuseAddr(reuseAddr); opts.noLinger(noLinger); opts.keepAlive(keepAlive); opts.ipLowDelay(ipLowDelay); opts.tcpNoDelay(tcpNoDelay); opts.receiveBufferSize(receiveBufferSize); opts.sendBufferSize(sendBufferSize); } SocketOptions::SocketOptions() { init(*this); } void SocketOptions::print(std::ostream& s) const { s << "SocketOptions[" << "bindAddr=" << bindAddr_ << ", " << "listenBacklog=" << listenBacklog_ << ", " << "reusePort=" << reusePort_ << ", " << "reuseAddr=" << reuseAddr_ << ", " << "keepAlive=" << keepAlive_ << ", " << "noLinger=" << noLinger_ << ", " << "ipLowDelay=" << ipLowDelay_ << ", " << "tcpNoDelay=" << tcpNoDelay_ << ", " << "receiveBufferSize=" << receiveBufferSize_ << ", " << "sendBufferSize=" << sendBufferSize_ << "]" << std::endl; } SocketOptions SocketOptions::none() { return SocketOptions(); } SocketOptions SocketOptions::server() { return SocketOptions().reuseAddr(true).keepAlive(true); } SocketOptions SocketOptions::control() { return SocketOptions().keepAlive(true).ipLowDelay(true).tcpNoDelay(true); } SocketOptions SocketOptions::data() { return SocketOptions(); } std::ostream& operator<<(std::ostream& s, const SocketOptions& o) { o.print(s); return s; } } // namespace eckit::net eckit-2.0.7/src/eckit/net/NetUser.h0000664000175000017500000000157415161702250017223 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_NetUser_h #define eckit_NetUser_h #include "eckit/net/TCPSocket.h" #include "eckit/thread/Thread.h" namespace eckit { class Stream; namespace net { class NetUser : public Thread { public: NetUser(net::TCPSocket&); ~NetUser(); protected: TCPSocket protocol_; private: virtual void serve(Stream&, std::istream&, std::ostream&) = 0; void run() override; friend class NetServiceProcessControler; }; } // namespace net } // namespace eckit #endif eckit-2.0.7/src/eckit/net/TCPStream.cc0000664000175000017500000000221115161702250017563 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/TCPStream.h" namespace eckit::net { TCPStream::TCPStream(net::TCPSocket& socket) : socket_(socket) {} TCPStream::~TCPStream() {} void TCPStream::closeOutput() { socket_.closeOutput(); } //---------------------------------------------------------------------------------------------------------------------- // Tricky solution to be removed when 'mutable' is available // std::string TCPStreamBase::nonConstName() { std::ostringstream r; r << "TCPStream[" << socket() << "]"; return r.str(); } std::string TCPStreamBase::name() const { return ((TCPStreamBase*)this)->nonConstName(); } SharedTCPStream::SharedTCPStream(net::TCPSocket& s) : TCPStream(s) {} SharedTCPStream::~SharedTCPStream() {} } // namespace eckit::net eckit-2.0.7/src/eckit/net/TelnetUser.cc0000664000175000017500000000207115161702250020057 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/TelnetUser.h" #include "eckit/net/Telnet.h" #include "eckit/runtime/Monitor.h" namespace eckit::net { TelnetUser::TelnetUser(TCPSocket& protocol) : NetUser(protocol), from_(protocol_.remoteHost()) {} TelnetUser::~TelnetUser() {} void TelnetUser::serve(Stream&, std::istream& in, std::ostream& out) { Log::debug() << "Starting a telnet connection " << std::endl; Monitor::instance().kind("telnet"); Monitor::instance().name(from_); while (!stopped()) { out << "Telnet not supported any more (for now anyway)" << std::endl; break; } Log::info() << "Exiting telnet user loop..." << std::endl; } } // namespace eckit::net eckit-2.0.7/src/eckit/net/UDPServer.h0000664000175000017500000000243015161702250017445 0ustar alastairalastair/* * (C) Copyright 1996-2017 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_net_UDPServer_h #define eckit_net_UDPServer_h #include #include namespace eckit { class Buffer; namespace net { class UDPServer { public: // methods explicit UDPServer(int port); UDPServer(const UDPServer&) = delete; UDPServer& operator=(const UDPServer&) = delete; UDPServer(UDPServer&&) = delete; UDPServer& operator=(UDPServer&&) = delete; ~UDPServer(); size_t receive(void* buf, long length); size_t receive(eckit::Buffer&); protected: // methods void print(std::ostream& s) const; friend std::ostream& operator<<(std::ostream& s, const UDPServer& socket) { socket.print(s); return s; } std::string remoteHost(struct sockaddr_storage&) const; private: // members int port_; int socketfd_; }; } // namespace net } // namespace eckit #endif eckit-2.0.7/src/eckit/net/HttpHeader.h0000664000175000017500000000613015161702250017657 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino #ifndef HttpHeader_H #define HttpHeader_H #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/MemoryHandle.h" namespace eckit { namespace net { class TCPSocket; }; //---------------------------------------------------------------------------------------------------------------------- class HttpError : public Exception { int status_; public: enum { OK = 200, CREATED = 201, ACCEPTED = 202, NO_CONTENT = 204, SEE_OTHER = 303, BAD_REQUEST = 400, UNAUTHORIZED = 401, NOT_FOUND = 404, CONTENT_TOO_LARGE = 413, NOT_IMPLEMENTED = 501, INTERNAL_SERVER_ERROR = 500, }; public: HttpError(int status, const std::string& msg = "HttpError") : Exception(msg), status_(status) {} int status() const { return status_; } }; class HttpHeader { public: // methods HttpHeader(); HttpHeader(net::TCPSocket&); HttpHeader(const HttpHeader&) = delete; HttpHeader& operator=(const HttpHeader&) = delete; HttpHeader(HttpHeader&&) = delete; HttpHeader& operator=(HttpHeader&&) = delete; ~HttpHeader(); HttpHeader& operator=(std::map >&); void length(const long); long contentLength() const; void type(const std::string&); void status(const long, const std::string& message = ""); void authenticate(const std::string&); bool authenticated() const; void forward(const std::string&); void dontCache(); void retryAfter(long); const std::string& type() const; const std::string& getHeader(const std::string&) const; void setHeader(const std::string&, const std::string&); void content(const char*, long); std::string content() const; void checkForStatus() const; protected: // methods void print(std::ostream&) const; private: // members std::string version_; long statusCode_; long contentLength_; std::string message_; bool received_; struct compare { bool operator()(const std::string&, const std::string&) const; }; using Map = std::map; Map header_; eckit::MemoryHandle content_; private: // methods friend std::ostream& operator<<(std::ostream& s, const HttpHeader& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit #endif eckit-2.0.7/src/eckit/net/UDPServer.cc0000664000175000017500000000711615161702250017611 0ustar alastairalastair/* * (C) Copyright 1996-2017 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include #include #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/log/Bytes.h" #include "eckit/net/UDPServer.h" #include "eckit/utils/Translator.h" namespace eckit::net { static void* get_sockaddr(struct sockaddr* sa) { if (sa->sa_family == AF_INET) { return &(((struct sockaddr_in*)sa)->sin_addr); } return &(((struct sockaddr_in6*)sa)->sin6_addr); } UDPServer::UDPServer(int port) : port_(port), socketfd_(0) { struct addrinfo hints; struct addrinfo* servinfo; struct addrinfo* addr; ::memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_DGRAM; hints.ai_flags = AI_PASSIVE; // use local IP Translator toStr; int err = 0; if ((err = ::getaddrinfo(nullptr, toStr(port_).c_str(), &hints, &servinfo)) != 0) { std::ostringstream msg; msg << "getaddrinfo failed in UDPServer with " << " port=" << port << " -- " << ::gai_strerror(err); throw FailedSystemCall(msg.str(), Here()); } // loop through all the addrinfo results and make a socket and then bind() for (addr = servinfo; addr != nullptr; addr = addr->ai_next) { if ((socketfd_ = ::socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol)) == -1) { continue; } if (::bind(socketfd_, addr->ai_addr, addr->ai_addrlen) == -1) { ::close(socketfd_); Log::warning() << "UPDServer failed to bind() to socket " << socketfd_ << std::endl; continue; } break; } if (addr == nullptr) { std::ostringstream msg; msg << "UDPServer failed to create a socket"; throw FailedSystemCall(msg.str(), Here()); } ::freeaddrinfo(servinfo); Log::info() << *this << std::endl; } UDPServer::~UDPServer() { SYSCALL(::close(socketfd_)); } size_t UDPServer::receive(void* buffer, long length) { struct sockaddr_storage remote_addr; socklen_t addr_len = sizeof(remote_addr); Log::info() << "UDPServer waiting on recvfrom()" << std::endl; ssize_t received = 0; if ((received = ::recvfrom(socketfd_, buffer, length, 0, (struct sockaddr*)&remote_addr, &addr_len)) == -1) { std::ostringstream msg; msg << "UDPServer port " << port_ << " error on recvfrom socket " << socketfd_; throw FailedSystemCall(msg.str(), Here()); } Log::info() << "Received messaged from " << remoteHost(remote_addr) << std::endl; return received; } size_t UDPServer::receive(eckit::Buffer& buffer) { return receive(buffer, buffer.size()); } void UDPServer::print(std::ostream& s) const { s << "UDPServer[port=" << port_ << ",socketfd=" << socketfd_ << "]"; } std::string UDPServer::remoteHost(struct sockaddr_storage& remote_addr) const { char inet6[INET6_ADDRSTRLEN]; std::string r = ::inet_ntop(remote_addr.ss_family, get_sockaddr((struct sockaddr*)&remote_addr), inet6, sizeof(inet6)); return r; } } // namespace eckit::net eckit-2.0.7/src/eckit/net/NetMask.h0000664000175000017500000000173315161702250017175 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @date Dec 2019 #ifndef eckit_net_NetMask_H #define eckit_net_NetMask_H #include #include #include "eckit/net/IPAddress.h" namespace eckit::net { class NetMask { public: // methods explicit NetMask(const std::string&); bool contains(const IPAddress&) const; private: // methods IPAddress network_; size_t bits_; void print(std::ostream& os) const; friend std::ostream& operator<<(std::ostream& os, const NetMask& ep) { ep.print(os); return os; } }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/Telnet.h0000664000175000017500000000172215161702250017064 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File Telnet.h // Baudouin Raoult - ECMWF Oct 96 #ifndef eckit_Telnet_h #define eckit_Telnet_h #include "eckit/net/NetService.h" namespace eckit::net { class Telnet : public NetService { public: // -- Contructors Telnet(int); // -- Destructor ~Telnet(); private: // No copy allowed Telnet(const Telnet&); Telnet& operator=(const Telnet&); // -- Overridden methods // From NetService virtual NetUser* newUser(net::TCPSocket&) const; virtual std::string name() const { return "telnet"; } }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/TCPClient.cc0000664000175000017500000000246715161702250017563 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/TCPClient.h" #include "eckit/config/Resource.h" #include "eckit/net/Endpoint.h" namespace eckit::net { TCPClient::TCPClient(const SocketOptions& options) : TCPSocket(), port_(0), options_(options) {} TCPClient::TCPClient(int port, const SocketOptions& options) : TCPSocket(), port_(port), options_(options) {} TCPClient::~TCPClient() {} void TCPClient::bind() { if (socket_ == -1) { socket_ = createSocket(port_, options_); } } void TCPClient::print(std::ostream& s) const { s << "TCPClient[" << "port=" << port_ << ",options=" << options_; TCPSocket::print(s); s << "]"; } /// @note TCPClient::connect(host, port, retries, timeout) is found in TCPSocket.cc TCPSocket& TCPClient::connect(const net::Endpoint& endpoint, int retries, int timeout, int retryDelay) { return connect(endpoint.hostname(), endpoint.port(), retries, timeout, retryDelay); } } // namespace eckit::net eckit-2.0.7/src/eckit/net/NetMask.cc0000664000175000017500000000267415161702250017340 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/NetMask.h" #include #include #include "eckit/exception/Exceptions.h" #include "eckit/net/IPAddress.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" // Cray C++ compiler should not try to optimize this code #if _CRAYC #pragma _CRI noopt #endif namespace eckit::net { static Translator s2l; // Not the most efficient or elegant, but should not affect performances static std::string parse(const std::string& cidr, size_t n) { static Tokenizer parse("/"); std::vector v; parse(cidr, v); ASSERT(v.size() == 2); return v[n]; } NetMask::NetMask(const std::string& cidr) : network_(parse(cidr, 0)), bits_(s2l(parse(cidr, 1))) { ASSERT(bits_ > 0); ASSERT(bits_ < 32); } bool NetMask::contains(const IPAddress& addr) const { return !((addr.address().s_addr ^ network_.address().s_addr) & htonl(0xFFFFFFFFu << (32 - bits_))); } void NetMask::print(std::ostream& s) const { s << network_ << '/' << bits_; } } // namespace eckit::net eckit-2.0.7/src/eckit/net/TCPClient.h0000664000175000017500000000307515161702250017421 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Jun 96 #ifndef eckit_TCPClient_h #define eckit_TCPClient_h #include "eckit/net/TCPSocket.h" namespace eckit::net { class Endpoint; class TCPClient : public TCPSocket { public: TCPClient(const SocketOptions& options = SocketOptions::none()); TCPClient(int port, const SocketOptions& options = SocketOptions::none()); TCPClient(TCPClient&) = delete; TCPClient& operator=(TCPClient&) = delete; TCPClient(TCPClient&&) = delete; TCPClient& operator=(TCPClient&&) = delete; ~TCPClient(); virtual TCPSocket& connect(const std::string& host, int port, int retries = 5, int timeout = 0, int retryDelay = 5); virtual TCPSocket& connect(const net::Endpoint& endpoint, int retries = 5, int timeout = 0, int retryDelay = 5); protected: // methods void print(std::ostream& s) const override; void buildSockAddress(); private: // members int port_; SocketOptions options_; private: // methods void bind() override; std::string bindingAddress() const override { return options_.bindAddress(); } }; } // namespace eckit::net #endif // TCPClient_H eckit-2.0.7/src/eckit/net/SocketOptions.h0000664000175000017500000000757715161702250020453 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_net_SocketOptions_h #define eckit_net_SocketOptions_h /// @author Baudouin Raoult /// @author Tiago Quintino /// @date Jan 2020 #include #include namespace eckit::net { struct SocketOptions { static SocketOptions none(); static SocketOptions server(); static SocketOptions control(); static SocketOptions data(); SocketOptions& bindAddress(const std::string& addr) { bindAddr_ = addr; return *this; } SocketOptions& listenBacklog(int backlog) { listenBacklog_ = backlog; return *this; } SocketOptions& receiveBufferSize(int size) { receiveBufferSize_ = size; return *this; } SocketOptions& sendBufferSize(int size) { sendBufferSize_ = size; return *this; } SocketOptions& reusePort(bool v) { reusePort_ = v; return *this; } SocketOptions& reuseAddr(bool v) { reuseAddr_ = v; return *this; } SocketOptions& noLinger(bool v) { noLinger_ = v; return *this; } SocketOptions& keepAlive(bool v) { keepAlive_ = v; return *this; } SocketOptions& ipLowDelay(bool v) { ipLowDelay_ = v; return *this; } SocketOptions& tcpNoDelay(bool v) { tcpNoDelay_ = v; return *this; } bool reusePort() const { return reusePort_; } bool reuseAddr() const { return reuseAddr_; } bool keepAlive() const { return keepAlive_; } bool noLinger() const { return noLinger_; } bool ipLowDelay() const { return ipLowDelay_; } bool tcpNoDelay() const { return tcpNoDelay_; } int listenBacklog() const { return listenBacklog_; } int receiveBufferSize() const { return receiveBufferSize_; } int sendBufferSize() const { return sendBufferSize_; } std::string bindAddress() const { return bindAddr_; } void print(std::ostream& s) const; private: SocketOptions(); friend std::ostream& operator<<(std::ostream& s, const SocketOptions& socket); private: /// Binding address for this socket std::string bindAddr_ = ""; /// Value to pass as backlog to ::listen() sockets int listenBacklog_ = 5; /// size in bytes of the receive buffer int receiveBufferSize_ = 0; /// size in bytes of the send buffer int sendBufferSize_ = 0; /// SO_REUSEPORT is useful if multiple threads want to bind to the same port and OS handles load balancing /// otherwise better not to set it. bool reusePort_ = false; /// SO_REUSEADDRESS tells OS to skip TIME_WAIT of 2MSL before rebinding to same address:port /// This allows fast restart of services, at the added risk of duplicated/stray packets in route /// from previous closed connections messing up new connections. This is likely under high network contention and/or /// very short connections quickly reuse the available ports. /// Set to false when openeing ephemeral ports/connections. bool reuseAddr_ = false; /// Enable sending of keep-alive messages bool keepAlive_ = false; /// Do the close in the background (don't wait for queued messages) and return immedietly on close() bool noLinger_ = false; /// sets IP_TOS to IPTOS_LOWDELAY to minimize delays for interactive traffic (small messages) as per manpage ip(7) bool ipLowDelay_ = false; /// bypass Nagle Delays by disabling Nagle's algorithm and send the data as soon as it's available bool tcpNoDelay_ = false; }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/Port.cc0000664000175000017500000000125415161702250016713 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/Port.h" #include "eckit/config/Resource.h" namespace eckit::net { Port::Port(const std::string& name, int port) : port_(port) { int offset = Resource("portOffset", 0); port_ = Resource(name + "Port", port) + offset; } } // namespace eckit::net eckit-2.0.7/src/eckit/net/NetUser.cc0000664000175000017500000000172415161702250017356 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/NetUser.h" #include "eckit/io/SockBuf.h" #include "eckit/net/TCPStream.h" namespace eckit::net { NetUser::NetUser(net::TCPSocket& protocol) : protocol_(protocol) { Log::status() << "New connection from " << protocol_.remoteHost() << std::endl; } NetUser::~NetUser() { Log::status() << "End connection from " << protocol_.remoteHost() << std::endl; } void NetUser::run() { SockBuf buf(protocol_); std::ostream out(&buf); std::istream in(&buf); InstantTCPStream stream(protocol_); serve(stream, in, out); } } // namespace eckit::net eckit-2.0.7/src/eckit/net/Endpoint.h0000664000175000017500000000423515161702250017413 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @author Tiago Quintino /// @date May 2019 #ifndef eckit_net_Endpoint_H #define eckit_net_Endpoint_H #include #include namespace eckit { class Stream; namespace net { //---------------------------------------------------------------------------------------------------------------------- class Endpoint { public: // methods Endpoint(const std::string&); // parses the std::string formated as hostname:port Endpoint(const std::string& host, int port); Endpoint(Stream& s); Endpoint(); virtual ~Endpoint() {} virtual const std::string& hostname() const { return host_; } const std::string& host() const { return host_; } int port() const { return port_; } operator std::string() const { return hostname() + ":" + std::to_string(port_); } bool operator==(const net::Endpoint& other) const; bool operator!=(const net::Endpoint& other) const { return !(*this == other); } void print(std::ostream& os) const; void encode(Stream& s) const; protected: // members std::string host_; int port_; private: // methods void validate() const; friend std::ostream& operator<<(std::ostream& os, const net::Endpoint& ep) { ep.print(os); return os; } friend Stream& operator<<(Stream& s, const net::Endpoint& ep) { ep.encode(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace net } // namespace eckit template <> struct std::hash { std::size_t operator()(const eckit::net::Endpoint& endpoint) const noexcept { const std::string& e = endpoint; return std::hash{}(e); } }; #endif eckit-2.0.7/src/eckit/net/ProxiedTCPServer.cc0000664000175000017500000000326215161702250021140 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include "eckit/net/ProxiedTCPServer.h" #include "eckit/io/Select.h" #include "eckit/log/Log.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ProxiedTCPServer::ProxiedTCPServer(int port, const net::SocketOptions& options) : net::TCPServer(port, options) {} ProxiedTCPServer::~ProxiedTCPServer() {} // Accept a client net::TCPSocket& ProxiedTCPServer::accept(const std::string& message, int timeout, bool* connected) { net::TCPSocket& socket = TCPServer::accept(message, timeout, connected); // Strip http-header char c; unsigned long x = 0; unsigned long end = ('\r' << 24L) | ('\n' << 16L) | ('\r' << 8L) | '\n'; while (socket.read(&c, 1) == 1) { x <<= 8L; x |= c; x &= 0xffffffff; if (x == end) { return socket; } } throw SeriousBug("ProxiedTCPServer: invalid header"); } void ProxiedTCPServer::print(std::ostream& s) const { s << "ProxiedTCPServer["; TCPServer::print(s); s << "]"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/net/MultiSocket.cc0000664000175000017500000001325115161702250020232 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/net/MultiSocket.h" #include "eckit/net/TCPClient.h" #include "eckit/net/TCPServer.h" #include "eckit/net/TCPStream.h" #include "eckit/utils/MD5.h" namespace eckit::net { const size_t VERSION = 1; // Server MultiSocket::MultiSocket(int port) { accept_ = new TCPServer(port); ASSERT(accept_); } MultiSocket::MultiSocket(size_t streams, size_t messageSize) : streams_(streams), messageSize_(messageSize) { ASSERT(streams > 0); ASSERT(messageSize > 0); } MultiSocket::~MultiSocket() { cleanup(); } void MultiSocket::cleanup() { delete accept_; accept_ = nullptr; for (TCPSocket* s : sockets_) { delete s; } sockets_.clear(); } void MultiSocket::close() { if (accept_) { select_.remove(*accept_); accept_->close(); } for (TCPSocket* s : sockets_) { select_.remove(*s); s->close(); } cleanup(); } long MultiSocket::write(const void* buf, long length) { // Log::info() << "MultiSocket::write length=" << length << std::endl; ASSERT(messageSize_); ASSERT(bytesWritten_ < messageSize_); long written = 0; const char* p = reinterpret_cast(buf); while (length > 0) { long len = std::min(long(messageSize_ - bytesWritten_), length); // Log::info() << "MultiSocket::write socket=" << writeSocket_ << " len=" << len << std::endl; len = sockets_[writeSocket_]->write(p, len); if (len > 0) { written += len; bytesWritten_ += len; if (bytesWritten_ == messageSize_) { writeSocket_ += 1; writeSocket_ %= streams_; bytesWritten_ = 0; } length -= len; p += len; } else { return written ? written : len; } } return written; } long MultiSocket::read(void* buf, long length) { // Log::info() << "MultiSocket::read length=" << length << std::endl; ASSERT(messageSize_); ASSERT(bytesRead_ < messageSize_); long read = 0; char* p = reinterpret_cast(buf); while (length > 0) { long len = std::min(long(messageSize_ - bytesRead_), length); // Log::info() << "MultiSocket::read socket=" << readSocket_ << " len=" << len << std::endl; len = sockets_[readSocket_]->read(p, len); if (len > 0) { read += len; bytesRead_ += len; if (bytesRead_ == messageSize_) { readSocket_ += 1; readSocket_ %= streams_; bytesRead_ = 0; } length -= len; p += len; } else { return read ? read : len; } } return read; } MultiSocket& MultiSocket::connect(const std::string& host, int port, int retries, int timeout) { ASSERT(!accept_); ASSERT(messageSize_); ASSERT(streams_); MD5 md5; char hostname[256] = {}; SYSCALL(::gethostname(hostname, sizeof(hostname) - 1)); md5.add(std::string(hostname)); md5.add(::time(nullptr)); md5.add(::getpid()); md5.add(reinterpret_cast(this)); id_ = md5.digest(); for (size_t i = 0; i < streams_; ++i) { std::unique_ptr p(new TCPClient()); p->bufferSize(bufferSize_); p->connect(host, port, retries, timeout); InstantTCPStream s(*p); s << VERSION; s << id_; s << i; s << streams_; s << messageSize_; sockets_.push_back(p.release()); } return *this; } MultiSocket& MultiSocket::accept() { ASSERT(accept_); ASSERT(sockets_.size() == 0); // Reset messageSize_ = 0; id_ = ""; streams_ = 0; size_t count = 0; size_t i = 0; for (;;) { std::unique_ptr p(new TCPSocket(accept_->accept())); InstantTCPStream s(*p); size_t version = 0; s >> version; ASSERT(version == VERSION); size_t streams = 0; size_t messageSize = 0; std::string id; s >> id; s >> i; s >> streams; s >> messageSize; if (id_.size()) { ASSERT(id_ == id); } else { Log::info() << "MultiSocket::accept id=" << id << std::endl; id_ = id; } if (streams_) { ASSERT(streams_ == streams); } else { streams_ = streams; sockets_ = std::vector(streams_, nullptr); Log::info() << "MultiSocket::accept streams=" << streams << std::endl; } if (messageSize_) { ASSERT(messageSize_ == messageSize); } else { messageSize_ = messageSize; Log::info() << "MultiSocket::accept messageSize=" << messageSize << std::endl; } ASSERT(i < streams_); ASSERT(sockets_[i] == nullptr); sockets_[i] = p.release(); count++; if (count == streams_) { break; } } return *this; } MultiSocket::MultiSocket(MultiSocket& other) : streams_(other.streams_), messageSize_(other.messageSize_) { ASSERT(messageSize_); std::swap(sockets_, other.sockets_); ASSERT(sockets_.size() == streams_); } } // namespace eckit::net eckit-2.0.7/src/eckit/net/MultiSocket.h0000664000175000017500000000405615161702250020077 0ustar alastairalastair/* * (C) Copyright 2021- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_net_MultiSocket_h #define eckit_net_MultiSocket_h #include #include #include #include "eckit/io/Select.h" namespace eckit::net { class TCPServer; class TCPSocket; class MultiSocket { public: MultiSocket(size_t streams, size_t messageSize); // Client MultiSocket(int port); // Server ~MultiSocket(); MultiSocket& accept(); MultiSocket& connect(const std::string& host, int port, int retries = 5, int timeout = 0); MultiSocket(MultiSocket&); // Take ownership long write(const void* buf, long length); long read(void* buf, long length); // close sockets void close(); // peer name in_addr remoteAddr() const; const std::string& remoteHost() const; int remotePort() const; in_addr localAddr() const; const std::string& localHost() const; int localPort() const; void bufferSize(int n) { bufferSize_ = n; } void closeOutput(); void closeInput(); void debug(bool on); private: // methods MultiSocket(const MultiSocket&); MultiSocket& operator=(const MultiSocket&); void print(std::ostream& s) const; void cleanup(); // --- Select select_; TCPServer* accept_ = nullptr; std::vector sockets_; size_t streams_ = 0; int readSocket_ = 0; size_t bytesRead_ = 0; int writeSocket_ = 0; size_t bytesWritten_ = 0; size_t messageSize_ = 0; std::string id_; int bufferSize_ = 0; friend std::ostream& operator<<(std::ostream& s, const MultiSocket& socket) { socket.print(s); return s; } }; } // namespace eckit::net #endif eckit-2.0.7/src/eckit/net/IPAddress.cc0000664000175000017500000000405315161702250017605 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/IPAddress.h" #include "eckit/exception/Exceptions.h" #include #include #include #include #include #include #include namespace eckit::net { IPAddress::IPAddress(const std::string& addr) { if (inet_aton(addr.c_str(), &address_) == 0) { std::ostringstream os; os << "Invalid IP address [" << addr << "]"; throw eckit::SeriousBug(os.str()); } } IPAddress::IPAddress(const char* addr) { if (inet_aton(addr, &address_) == 0) { std::ostringstream os; os << "Invalid IP address [" << addr << "]"; throw eckit::SeriousBug(os.str()); } } void IPAddress::print(std::ostream& os) const { os << inet_ntoa(address_); } std::string IPAddress::asString() const { return inet_ntoa(address_); } bool IPAddress::operator==(const IPAddress& other) const { return ::memcmp(&address_, &other.address_, sizeof(address_)) == 0; } IPAddress IPAddress::hostAddress(const std::string& hostname) { struct hostent* hostEntry = gethostbyname(hostname.c_str()); ASSERT(hostEntry); return IPAddress(inet_ntoa(*((struct in_addr*)hostEntry->h_addr_list[0]))); } IPAddress IPAddress::myIPAddress() { static bool done = false; static IPAddress mine("255.255.255.255"); if (!done) { char hostname[256] = {}; SYSCALL(::gethostname(hostname, sizeof(hostname) - 1)); struct hostent* entry = gethostbyname(hostname); ASSERT(entry); mine = IPAddress(inet_ntoa(*((struct in_addr*)entry->h_addr_list[0]))); done = true; } return mine; } } // namespace eckit::net eckit-2.0.7/src/eckit/net/Endpoint.cc0000664000175000017500000000362415161702250017552 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/Endpoint.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/serialisation/Stream.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" namespace eckit::net { //---------------------------------------------------------------------------------------------------------------------- Endpoint::Endpoint(const std::string& s) { Tokenizer tokenize(":"); std::vector tokens; tokenize(s, tokens); ASSERT(tokens.size() == 2); host_ = tokens[0]; port_ = Translator()(tokens[1]); validate(); } Endpoint::Endpoint(const std::string& host, int port) : host_(host), port_(port) { validate(); } Endpoint::Endpoint(Stream& s) { s >> host_; s >> port_; validate(); } Endpoint::Endpoint() : port_(0) { validate(); } bool Endpoint::operator==(const net::Endpoint& other) const { return (port_ == other.port_ && hostname() == other.hostname()); } void Endpoint::print(std::ostream& os) const { os << hostname() << ":" << port_; } void Endpoint::encode(Stream& s) const { s << host_; s << port_; } void Endpoint::validate() const { // IP ranges are valid 1 - 65535 if (port_ < 0 or port_ > 65535) { std::ostringstream msg; msg << "Invalid port number " << port_; throw eckit::BadValue(msg.str(), Here()); } } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::net eckit-2.0.7/src/eckit/net/ProxiedTCPClient.cc0000664000175000017500000000350315161702250021106 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/ProxiedTCPClient.h" #include "eckit/net/HttpHeader.h" namespace eckit { //---------------------------------------------------------------------------------------------------------------------- ProxiedTCPClient::ProxiedTCPClient(const std::string& proxyHost, int proxyPort, int port) : TCPClient(port), proxy_(proxyHost, proxyPort) {} ProxiedTCPClient::~ProxiedTCPClient() {} net::TCPSocket& ProxiedTCPClient::connect(const std::string& host, int port, int retries, int timeout, int retryDelay) { net::TCPSocket& socket = TCPClient::connect(proxy_.hostname(), proxy_.port(), retries, timeout, retryDelay); socket.debug(debug_.on); const char* CRLF = "\r\n"; std::ostringstream oss; oss << "CONNECT " << host << ":" << port << " HTTP/1.0" << CRLF; oss << "User-agent: eckit/1.0" << CRLF; oss << CRLF; std::string request(oss.str()); auto len = long(request.size()); ASSERT(socket.write(&request[0], len) == len); // Strip http-header HttpHeader header(socket); std::cout << std::endl << header << std::endl << std::endl; header.checkForStatus(); return socket; } void ProxiedTCPClient::print(std::ostream& s) const { s << "ProxiedTCPClient[" << "proxy=" << proxy_ << ","; TCPClient::print(s); s << "]"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit eckit-2.0.7/src/eckit/net/Telnetable.cc0000664000175000017500000000124715161702250020050 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/net/Telnetable.h" #include "eckit/net/Telnet.h" namespace eckit::net { Telnetable::Telnetable(int port) : telnet_(new Telnet(port)) { if (port != 0) { telnet_.start(); } } Telnetable::~Telnetable() { telnet_.stop(); } } // namespace eckit::net eckit-2.0.7/src/eckit/net/TCPSocket.h0000664000175000017500000000734615161702250017440 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_net_TCPSocket_h #define eckit_net_TCPSocket_h #include #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/net/SocketOptions.h" #include "eckit/utils/Hash.h" namespace eckit::net { /// @note this class calls sets a handler to ignore SIGPIPE class TCPSocket { public: // types class UnknownHost : public Exception { public: explicit UnknownHost(const std::string&); }; public: // methods TCPSocket(); // From an existing TCPSocket (see TCPServer::accept) /// @warning /// **** NOTE: copying gives ownership of the socket to new object /// **** Beware of 'slicing', i.e copying subclasses. TCPSocket(net::TCPSocket&); virtual ~TCPSocket(); // See note on copy constructor TCPSocket& operator=(net::TCPSocket&); long write(const void* buf, long length) const; /// Read from a TCP socket /// /// \param buf The buffer to read into /// \param length The maximum number of bytes to read /// /// **Configuration flags** /// \arg **useSelectOnTCPSocket** (*bool*): use select for improved resilience /// on flaky connections /// \arg **socketSelectTimeout** (*long*): timeout in seconds for the select /// (only if **useSelectOnTCPSocket** is enabled) long read(void* buf, long length) const; long rawRead(void*, long); // Non-blocking version bool isConnected() const { return socket_ != -1; } bool stillConnected() const; // close sockets virtual void close(); // peer name in_addr remoteAddr() const; const std::string& remoteHost() const; int remotePort() const; in_addr localAddr() const; const std::string& localHost() const; int localPort() const; void bufferSize(int n) { receiveBufferSize(n); sendBufferSize(n); } void receiveBufferSize(int n) { receiveBufferSize_ = n; } void sendBufferSize(int n) { sendBufferSize_ = n; } virtual int socket(); void closeOutput(); void closeInput(); void debug(bool on); public: // class methods static std::string addrToHost(in_addr); static in_addr hostToAddr(const std::string&); static std::string hostName(const std::string& h, bool full = false); /// @note uses sigaction to ignore SIGPIPE static void register_ignore_sigpipe(); protected: // members int socket_; // i/o socket int localPort_; // effective port int remotePort_; // remote port std::string remoteHost_; // remote host in_addr remoteAddr_; // remote ip adress std::string localHost_; // local host in_addr localAddr_; // local ip adress int receiveBufferSize_ = 0; int sendBufferSize_ = 0; // Debug struct { bool on{false}; mutable bool newline{true}; mutable char mode{0}; } debug_; protected: // methods int createSocket(int port, const SocketOptions& options); virtual void print(std::ostream& s) const; private: // methods /// @pre socket must be made virtual void bind(); virtual std::string bindingAddress() const; friend std::ostream& operator<<(std::ostream& s, const TCPSocket& socket) { socket.print(s); return s; } }; std::ostream& operator<<(std::ostream&, in_addr); } // namespace eckit::net #endif eckit-2.0.7/src/eckit/sql/0000775000175000017500000000000015161702250015467 5ustar alastairalastaireckit-2.0.7/src/eckit/sql/SQLOutput.cc0000664000175000017500000000172515161702250017663 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLOutput.h" #include namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SQLOutput::SQLOutput() {} SQLOutput::~SQLOutput() {} void SQLOutput::preprepare(SQLSelect&) {} bool SQLOutput::cachedNext() { return false; } void SQLOutput::print(std::ostream& s) const { s << "SQLOutput" << std::endl; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLOrderOutput.h0000664000175000017500000000437415161702250020524 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// @date Nov 2011 /// @date Aug 2018 #ifndef eckit_sql_SQLOrderOutput_H #define eckit_sql_SQLOrderOutput_H #include #include "eckit/sql/SQLOutput.h" #include "eckit/sql/expression/OrderByExpressions.h" #include "eckit/sql/expression/SQLExpressions.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLOrderOutput : public SQLOutput { public: SQLOrderOutput(SQLOutput& output, const std::pair>& by); ~SQLOrderOutput() override; private: // methods void print(std::ostream&) const override; // -- Members SQLOutput& output_; std::pair> by_; using SortedResults = std::map>; SortedResults sortedResults_; std::vector byIndices_; // -- Overridden methods void reset() override; void flush() override; /// OrderBy builds a std::map of sorted results. Now we start outputting them. bool cachedNext() override; bool output(const expression::Expressions&) override; void preprepare(SQLSelect&) override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; unsigned long long count() override; // Overridden (and removed) functions void outputReal(double, bool) override; void outputDouble(double, bool) override; void outputInt(double, bool) override; void outputUnsignedInt(double, bool) override; void outputString(const char*, size_t, bool) override; void outputBitfield(double, bool) override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/Environment.h0000664000175000017500000000225615161702250020151 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_sql_Environment_H #define eckit_sql_Environment_H #include #include "eckit/sql/SQLTable.h" #include "eckit/sql/SelectOneTable.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLTableIterator; struct Environment { Environment(SortedTables::iterator it) : tableIterator(it) {} ~Environment() {} const SelectOneTable& table() const { return **tableIterator; } // n.b. ODB-357 and performance issues. SortedTables::iterator tableIterator; std::unique_ptr cursor; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLParser.h0000664000175000017500000000372415161702250017462 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLParser.h // Baudouin Raoult - ECMWF Mar 98 // Piotr Kuchta - ECMWF Mar 2012 #ifndef eckit_sql_SQLParser_H #define eckit_sql_SQLParser_H #include #include "SQLOutputConfig.h" #include "eckit/exception/Exceptions.h" namespace eckit { class PathName; } namespace eckit { class DataHandle; } namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLDatabase; class SQLSession; class SyntaxError : public eckit::Exception { using eckit::Exception::Exception; }; struct ParseFrame { ParseFrame(const std::string& sql, const std::string& yypath); std::string inputString_; std::string yypath_; char* inputText_; char* inputEnd_; }; class SQLParser { public: static int line(); // static void parseString(SQLSession&, const std::string&, eckit::DataHandle*, SQLOutputConfig, bool // resetSession = true); static void parseString(SQLSession&, const std::string&, std::istream*, SQLOutputConfig, // const std::string& cvsDelimiter); static void parseString(SQLSession&, const std::string&); // static void parseString(SQLSession&, const std::string&, SQLDatabase&, SQLOutputConfig); static std::stack frames_; private: static void parseStringInternal(SQLSession&, const std::string&); static std::string cleanUpSQLText(const std::string&); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLOutput.h0000664000175000017500000000447115161702250017526 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 2003 #ifndef eckit_sql_SQLOutput_H #define eckit_sql_SQLOutput_H #include "eckit/sql/SQLOutputConfig.h" namespace eckit::sql { namespace expression { class Expressions; } class SQLSelect; //---------------------------------------------------------------------------------------------------------------------- class SQLOutput { public: SQLOutput(); SQLOutput(const SQLOutput&) = delete; SQLOutput& operator=(const SQLOutput&) = delete; SQLOutput(SQLOutput&&) = delete; SQLOutput& operator=(SQLOutput&&) = delete; virtual ~SQLOutput(); virtual void preprepare(SQLSelect&); virtual void prepare(SQLSelect&) = 0; virtual void updateTypes(SQLSelect&) {} virtual void cleanup(SQLSelect&) = 0; virtual void reset() = 0; virtual void flush() = 0; /// If an iterator in the chain has started caching output (e.g. the /// OrderBy iterator), start flushing those through. Returns true /// when row is output, false otherwise. virtual bool cachedNext(); virtual bool output(const expression::Expressions&) = 0; virtual void outputReal(double, bool) = 0; virtual void outputDouble(double, bool) = 0; virtual void outputInt(double, bool) = 0; virtual void outputUnsignedInt(double, bool) = 0; virtual void outputString(const char*, size_t, bool) = 0; virtual void outputBitfield(double, bool) = 0; virtual unsigned long long count() = 0; protected: virtual void print(std::ostream&) const; private: // -- Friends friend std::ostream& operator<<(std::ostream& s, const SQLOutput& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/Environment.cc0000664000175000017500000000137015161702250020303 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/Environment.h" #include "eckit/sql/SQLTable.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLSimpleOutput.cc0000664000175000017500000001154415161702250021035 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLSimpleOutput.h" #include "eckit/eckit.h" #include "eckit/log/Number.h" #include "eckit/sql/SQLSelect.h" #include namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SQLSimpleOutput::SQLSimpleOutput(const SQLOutputConfig& config, std::ostream& out) : out_(out), count_(0), config_(config) { out_ << std::fixed; } SQLSimpleOutput::~SQLSimpleOutput() {} void SQLSimpleOutput::print(std::ostream& s) const { s << "SQLSimpleOutput"; } std::ostream& SQLSimpleOutput::format(std::ostream& o, size_t i) const { o.width(columnWidths_[i]); return o << *columnAlignments_[i]; } void SQLSimpleOutput::reset() { count_ = 0; } void SQLSimpleOutput::flush() { out_ << std::flush; } bool SQLSimpleOutput::output(const expression::Expressions& results) { size_t n = results.size(); for (size_t i = 0; i < n; i++) { if (i) { out_ << config_.fieldDelimiter(); } currentColumn_ = i; results[i]->output(*this); } out_ << "\n"; count_++; return true; } template void SQLSimpleOutput::outputValue(double x, bool missing) { format(out_, currentColumn_); if (missing && !config_.doNotWriteNULL()) { out_ << "NULL"; } else { if (config_.fullPrecision()) { out_.precision(std::numeric_limits::digits10 + 2); out_ << std::fixed << static_cast(x); } else { out_ << static_cast(x); } } } void SQLSimpleOutput::outputReal(double x, bool missing) { outputValue(x, missing); } void SQLSimpleOutput::outputDouble(double x, bool missing) { outputValue(x, missing); } void SQLSimpleOutput::outputInt(double x, bool missing) { outputValue(x, missing); } void SQLSimpleOutput::outputUnsignedInt(double x, bool missing) { outputValue(x, missing); } void SQLSimpleOutput::outputString(const char* s, size_t len, bool missing) { format(out_, currentColumn_); if (missing && !config_.doNotWriteNULL()) { out_ << "NULL"; } else { std::ostringstream ss; ss << "'" << std::string(s, len) << "'"; out_ << ss.str(); } } void SQLSimpleOutput::outputBitfield(double x, bool missing) { if (!config_.displayBitfieldsBinary()) { outputUnsignedInt(x, missing); return; } format(out_, currentColumn_); if (missing && !config_.doNotWriteNULL()) { out_ << "NULL"; } else { std::stringstream ss; log::Number::printBinary(ss, static_cast(x)); out_ << ss.str(); } } void SQLSimpleOutput::prepare(SQLSelect& sql) { updateTypes(sql); printHeader(sql); } void SQLSimpleOutput::updateTypes(SQLSelect& sql) { bool first = columnWidths_.size() == 0; const expression::Expressions& columns(sql.output()); for (size_t i(0); i < columns.size(); i++) { const std::string& name(columns[i]->title()); const type::SQLType* type(columns[i]->type()); size_t width = config_.disableAlignmentOfColumns() ? 1 : std::max(type->width(), name.size()); if (first) { columnWidths_.push_back(width); columnAlignments_.push_back(type->format()); } else { columnWidths_[i] = std::max(width, columnWidths_[i]); columnAlignments_[i] = type->format(); } } } void SQLSimpleOutput::printHeader(SQLSelect& sql) { if (!config_.doNotWriteColumnNames()) { const expression::Expressions& columns(sql.output()); for (size_t i(0); i < columns.size(); i++) { const std::string& name(columns[i]->title()); const type::SQLType* type(columns[i]->type()); if (i) { out_ << config_.fieldDelimiter(); } format(out_, i); if (config_.outputFormat() != "wide") { out_ << name; } else { std::stringstream ss; ss << name << ":" << type->name(); out_ << ss.str(); } } out_ << "\n"; } } void SQLSimpleOutput::cleanup(SQLSelect& sql) {} unsigned long long SQLSimpleOutput::count() { return count_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SelectOneTable.cc0000664000175000017500000000176415161702250020637 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SelectOneTable.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- static bool nullBool = false; SelectOneTable::SelectOneTable(const SQLTable* table) : table_(table), offset_(0, nullBool), length_(0, nullBool), column_{nullptr}, table1_{nullptr}, table2_{nullptr}, order_(0) {} SelectOneTable::~SelectOneTable() {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLBitColumn.cc0000664000175000017500000000271215161702250020254 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLBitColumn.h" #include "eckit/eckit.h" #include "eckit/log/Log.h" #include "eckit/sql/type/SQLBitfield.h" using namespace eckit; namespace eckit::sql { SQLBitColumn::SQLBitColumn(const SQLColumn& column, const std::string& field) : SQLColumn(column), field_(field), mask_(0), shift_(0) { const type::SQLBitfield& t = dynamic_cast(type()); mask_ = t.mask(field); shift_ = t.shift(field); Log::info() << "here " << field << " mask=" << std::hex << mask_ << std::dec << " shift=" << shift_ << std::endl; } SQLBitColumn::~SQLBitColumn() {} void SQLBitColumn::rewind() { SQLColumn::rewind(); } double SQLBitColumn::next(bool& missing) { Log::info() << "SQLBitColumn::next: " << std::endl; unsigned long value = static_cast(SQLColumn::next(missing)); return (value >> shift_) & mask_; } void SQLBitColumn::advance(unsigned long n) { SQLColumn::advance(n); } void SQLBitColumn::print(std::ostream& s) const {} } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLMATCHSubquerySessionOutput.h0000664000175000017500000000404415161702250023403 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file SQLMATCHSubquerySessionOutput.h /// Piotr Kuchta - ECMWF Feb 09 #ifndef SQLMATCHSubquerySessionOutput_H #define SQLMATCHSubquerySessionOutput_H #include "eckit/sql/SQLOutput.h" class SelectIterator; namespace eckit { namespace sql { namespace expression { namespace function { class FunctionMATCH; } } // namespace expression } // namespace sql } // namespace eckit namespace eckit { namespace sql { class ReaderIterator; class SQLMATCHSubquerySessionOutput : public SQLOutput { public: SQLMATCHSubquerySessionOutput(odb::sql::expression::function::FunctionMATCH&); SQLMATCHSubquerySessionOutput(const SQLMATCHSubquerySessionOutput&); ~SQLMATCHSubquerySessionOutput() override; SQLMATCHSubquerySessionOutput& operator=(const SQLMATCHSubquerySessionOutput&); protected: void print(std::ostream&) const override; // -- Members odb::sql::expression::function::FunctionMATCH& f_; // -- Methods // None // -- Overridden methods virtual unsigned long long count(); virtual void size(int); virtual void reset(); void flush() override; virtual bool output(const expression::Expressions&); void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; virtual void outputReal(double, bool) { NOTIMP; }; virtual void outputDouble(double, bool) { NOTIMP; }; virtual void outputInt(double, bool) { NOTIMP; }; virtual void outputUnsignedInt(double, bool) { NOTIMP; }; virtual void outputString(double, bool) { NOTIMP; }; virtual void outputBitfield(double, bool) { NOTIMP; }; private: unsigned long long count_; }; } // namespace sql } // namespace eckit #endif eckit-2.0.7/src/eckit/sql/SQLIteratorOutput.cc0000664000175000017500000000327415161702250021376 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/SQLExpressions.h" namespace eckit { namespace sql { template SQLIteratorOutput::SQLIteratorOutput(T& it) : iterator_(it), count_(0) {} template SQLIteratorOutput::~SQLIteratorOutput() {} template void SQLIteratorOutput::print(std::ostream& s) const { s << "SQLIteratorOutput"; } template void SQLIteratorOutput::size(int) {} template void SQLIteratorOutput::reset() { count_ = 0; } template void SQLIteratorOutput::flush() {} template bool SQLIteratorOutput::output(const expression::Expressions& results) { size_t nCols = results.size(); /// ASSERT(nCols == iterator_.columns().size()); if (iterator_.isCachingRows()) iterator_.cacheRow(results); else for (size_t i = 0; i < nCols; i++) { bool missing = false; iterator_.data_[i] = results[i]->eval(missing /*=false*/); } count_++; return true; } template void SQLIteratorOutput::prepare(SQLSelect& sql) {} template void SQLIteratorOutput::cleanup(SQLSelect& sql) {} template unsigned long long SQLIteratorOutput::count() { return count_; } } // namespace sql } // namespace eckit eckit-2.0.7/src/eckit/sql/SQLDistinctOutput.cc0000664000175000017500000000612215161702250021361 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLDistinctOutput.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/expression/SQLExpressions.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SQLDistinctOutput::SQLDistinctOutput(SQLOutput& output) : output_(output) {} SQLDistinctOutput::~SQLDistinctOutput() {} void SQLDistinctOutput::print(std::ostream& s) const { s << "SQLDistinctOutput[" << output_ << "]"; } void SQLDistinctOutput::reset() { output_.reset(); seen_.clear(); } void SQLDistinctOutput::flush() { output_.flush(); } bool SQLDistinctOutput::cachedNext() { return output_.cachedNext(); } unsigned long long SQLDistinctOutput::count() { return output_.count(); } bool SQLDistinctOutput::output(const expression::Expressions& results) { // Get the data into a temporary buffer we can compare ASSERT(results.size() == offsets_.size()); for (size_t i = 0; i < results.size(); i++) { bool missing = false; results[i]->eval(&tmp_[offsets_[i]], missing); // What do we do with missing? Or has it been already evaluated somewhere before and it doesn't matter???... } if (seen_.find(tmp_) == seen_.end()) { seen_.insert(tmp_); return output_.output(results); } return false; } void SQLDistinctOutput::preprepare(SQLSelect& sql) { output_.preprepare(sql); } void SQLDistinctOutput::prepare(SQLSelect& sql) { output_.prepare(sql); updateTypes(sql); seen_.clear(); } void SQLDistinctOutput::updateTypes(SQLSelect& sql) { output_.updateTypes(sql); // How much space is needed to store each row of selected data offsets_.clear(); size_t offset = 0; for (const auto& column : sql.output()) { size_t colSizeBytes = column->type()->size(); ASSERT(colSizeBytes % 8 == 0); offsets_.push_back(offset); offset += colSizeBytes / 8; } // And buffers to do the storage tmp_.resize(offset); } void SQLDistinctOutput::cleanup(SQLSelect& sql) { output_.cleanup(sql); } // Direct output functions removed in distinct output void SQLDistinctOutput::outputReal(double, bool) { NOTIMP; } void SQLDistinctOutput::outputDouble(double, bool) { NOTIMP; } void SQLDistinctOutput::outputInt(double, bool) { NOTIMP; } void SQLDistinctOutput::outputUnsignedInt(double, bool) { NOTIMP; } void SQLDistinctOutput::outputString(const char*, size_t, bool) { NOTIMP; } void SQLDistinctOutput::outputBitfield(double, bool) { NOTIMP; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLSelect.h0000664000175000017500000001137415161702250017445 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 2003 #ifndef eckit_sql_SQLSelect_H #define eckit_sql_SQLSelect_H #include #include "eckit/filesystem/PathName.h" #include "eckit/sql/Environment.h" #include "eckit/sql/SQLOutput.h" #include "eckit/sql/SQLOutputConfig.h" #include "eckit/sql/SQLStatement.h" #include "eckit/sql/SelectOneTable.h" #include "eckit/sql/expression/OrderByExpressions.h" namespace eckit::sql { class SQLTableIterator; namespace expression::function { class FunctionROWNUMBER; class FunctionTHIN; } // namespace expression::function } // namespace eckit::sql namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLSelect : public SQLStatement { public: using ValueLookup = std::pair; SQLSelect(const Expressions& columns, const std::vector>& tables, std::shared_ptr where, SQLOutput& out, std::vector>&& ownedOutputs_ = std::vector>()); ~SQLSelect(); // -- Methods void prepareExecute(); unsigned long long process(); bool processOneRow(); void postExecute(); bool isAggregate() { return aggregate_; } ValueLookup& column(const std::string& name, const SQLTable*); const type::SQLType* typeOf(const std::string& name, const SQLTable*) const; const SQLTable& findTable(const std::string& name) const; void ensureFetch(const SQLTable& table, const std::string& columnName); Expressions output() const override; std::vector outputFiles() const; void outputFiles(const std::vector& files); const std::vector& tables() { return tables_; } expression::SQLExpression* where() { return where_.get(); } // -- Overridden methods unsigned long long execute() override; protected: void print(std::ostream&) const override; private: // No copy allowed SQLSelect(const SQLSelect&); SQLSelect& operator=(const SQLSelect&); /// Retrive a data pointer and offsets from the cursor (SQLTableIterator) associated /// with the given SQL table. void refreshCursorMetadata(SQLTable* table, SQLTableIterator& cursor); // -- Members Expressions select_; std::vector tables_; SortedTables sortedTables_; std::shared_ptr where_; std::shared_ptr simplifiedWhere_; // Cursors provide the environment for the iteration over the tables std::vector> cursors_; /// ownedOutputs allows us to create wrapping SQLOutputs with the same lifetime as the /// SQLSelect object. std::vector> ownedOutputs_; SQLOutput& output_; using AggregatedResults = std::map; AggregatedResults aggregatedResults_; AggregatedResults::iterator aggregatedResultsIterator_; // n.b. we don't use std::vector as you cannot take a reference to a single element. std::map values_; std::set allTables_; using TableMap = std::map; TableMap tablesToFetch_; unsigned long long count_; unsigned long long total_; unsigned long long skips_; bool aggregate_; bool mixedAggregatedAndScalar_; bool doOutputCached_; Expressions aggregated_; Expressions nonAggregated_; std::vector mixedResultColumnIsAggregated_; std::vector outputFiles_; // -- Methods void reset(); bool resultsOut(); bool writeOutput(); std::shared_ptr findAliasedExpression(const std::string& alias); bool processNextTableRow(size_t tableIndex); friend class expression::function::FunctionROWNUMBER; // needs access to count_ friend class expression::function::FunctionTHIN; // needs access to count_ friend std::ostream& operator<<(std::ostream& s, const SQLSelect& p) { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLDatabase.cc0000664000175000017500000000711115161702250020062 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLDatabase.h" #include #include #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/sql/expression/SQLExpression.h" #include "eckit/sql/type/SQLType.h" #include "eckit/utils/Tokenizer.h" using namespace eckit; namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SQLDatabase::SQLDatabase(const std::string& name) : name_(name) {} SQLDatabase::~SQLDatabase() {} void SQLDatabase::open() { Log::debug() << "Opening database: " << name_ << std::endl; } void SQLDatabase::close() { tablesByName_.clear(); } void SQLDatabase::addTable(SQLTable* table) { tablesByName_.emplace(table->name(), std::unique_ptr(table)); } void SQLDatabase::addImplicitTable(SQLTable* table) { implicitTables_.emplace_back(table); } void SQLDatabase::setLinks(const Links& links) { for (Links::const_iterator j = links.begin(); j != links.end(); ++j) { const std::string& from = (*j).first; const std::set& to = (*j).second; ASSERT(tablesByName_.find(from) != tablesByName_.end()); SQLTable& f = *tablesByName_[from]; for (std::set::const_iterator k = to.begin(); k != to.end(); ++k) { ASSERT(tablesByName_.find(*k) != tablesByName_.end()); SQLTable& t = *tablesByName_[*k]; f.addLinkTo(t); t.addLinkFrom(f); } } } SQLTable& SQLDatabase::defaultTable() { auto it = tablesByName_.find("defaultTable"); if (it == tablesByName_.end()) { if (tablesByName_.empty() && implicitTables_.size() != 0) { return *implicitTables_.back(); } throw UserError("No default table"); } return *(it->second); } std::vector> SQLDatabase::implicitTables() { std::vector> tables; for (std::unique_ptr& ptable : implicitTables_) { ASSERT(ptable); tables.push_back(*ptable); } return tables; } bool SQLDatabase::hasTable(const std::string& name) const { return tablesByName_.find(name) != tablesByName_.end(); } SQLTable& SQLDatabase::table(const std::string& nm) { auto it = tablesByName_.find(nm); ASSERT(it != tablesByName_.end()); return *(it->second); } void SQLDatabase::setVariable(const std::string& name, std::shared_ptr value) { variables_[name] = value; } std::shared_ptr SQLDatabase::getVariable(const std::string& name) const { Variables::const_iterator j = variables_.find(name); if (j == variables_.end()) { throw eckit::UserError("Undefined variable", name); } return j->second; } void SQLDatabase::setIncludePath(const std::string& includePath) { Tokenizer tokenize(":"); std::vector tokens; tokenize(includePath, tokens); copy(tokens.begin(), tokens.end(), back_inserter(includePath_)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLDistinctOutput.h0000664000175000017500000000545315161702250021231 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 2003 /// @date Aug 2018 #ifndef eckit_sql_SQLDistinctOutput_H #define eckit_sql_SQLDistinctOutput_H #include "eckit/sql/SQLOutput.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLDistinctOutput : public SQLOutput { // Comparator to ensure that all cases are well defined, even if we are representing non-double // data as doubles struct DoubleBitwiseLessComparator { bool operator()(const std::vector& lhs, const std::vector& rhs) const { if (lhs.size() != rhs.size()) { return lhs.size() < rhs.size(); } for (int i = 0; i < lhs.size(); ++i) { const void* p1 = &lhs[i]; const void* p2 = &rhs[i]; const uint64_t& punned1 = *reinterpret_cast(p1); const uint64_t& punned2 = *reinterpret_cast(p2); if (punned1 != punned2) { return punned1 < punned2; } } return false; } }; public: // methods SQLDistinctOutput(SQLOutput& output); ~SQLDistinctOutput() override; private: // methods void print(std::ostream&) const override; // -- Members SQLOutput& output_; std::set, DoubleBitwiseLessComparator> seen_; std::vector tmp_; std::vector offsets_; // -- Overridden methods void reset() override; void flush() override; bool cachedNext() override; bool output(const expression::Expressions&) override; void preprepare(SQLSelect&) override; void prepare(SQLSelect&) override; void updateTypes(SQLSelect&) override; void cleanup(SQLSelect&) override; unsigned long long count() override; // Overridden (and removed) functions void outputReal(double, bool) override; void outputDouble(double, bool) override; void outputInt(double, bool) override; void outputUnsignedInt(double, bool) override; void outputString(const char*, size_t, bool) override; void outputBitfield(double, bool) override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLIterator.h0000664000175000017500000000337315161702250020017 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLIterator // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLIterator_H #define SQLIterator_H #include "eckit/sql/type/SQLType.h" #include namespace eckit { namespace sql { // Forward declarations class SQLIterator { public: // void* operator new(size_t s) { return eckit::MemoryPool::fastAllocate(s); } // void *operator new(size_t s,void *p) { return p; } // void operator delete(void* p) { eckit::MemoryPool::fastDeallocate(p); } SQLIterator(const type::SQLType& type) : type_(type) {} virtual ~SQLIterator() {} const type::SQLType& type() const { return type_.get(); } void updateType(const type::SQLType& type) { type_ = std::cref(type); } virtual void rewind() = 0; virtual double next(bool& missing) = 0; virtual void advance(unsigned long) = 0; virtual void load() {} virtual void unload() {} virtual size_t dataSizeDoubles() const { return 1; } protected: std::reference_wrapper type_; virtual void print(std::ostream&) const = 0; private: // No copy allowed SQLIterator(const SQLIterator&); SQLIterator& operator=(const SQLIterator&); friend std::ostream& operator<<(std::ostream& s, const SQLIterator& p) { p.print(s); return s; } }; } // namespace sql } // namespace eckit #endif eckit-2.0.7/src/eckit/sql/SQLBitColumn.h0000664000175000017500000000234015161702250020113 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLBitColumn.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLBitColumn_H #define SQLBitColumn_H #include "eckit/sql/SQLColumn.h" namespace eckit::sql { class SQLBitColumn : public SQLColumn { public: SQLBitColumn(const SQLColumn&, const std::string&); ~SQLBitColumn(); private: // No copy allowed SQLBitColumn(const SQLBitColumn&); SQLBitColumn& operator=(const SQLBitColumn&); std::string field_; unsigned long mask_; unsigned long shift_; // -- Overridden methods // From ODBIterator void rewind() override; double next(bool& missing) override; void advance(unsigned long) override; void print(std::ostream&) const override; // friend std::ostream& operator<<(std::ostream& s,const SQLBitColumn& p) // { p.print(s); return s; } }; } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLTable.h0000664000175000017500000001062715161702250017255 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart // @date Dec 03 #ifndef eckit_sql_SQLTable_H #define eckit_sql_SQLTable_H #include #include #include "eckit/filesystem/PathName.h" #include "eckit/sql/SQLTypedefs.h" #include "eckit/sql/type/SQLType.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- // class SQLFile; class SQLColumn; class SQLDatabase; class SQLTableIterator { public: virtual ~SQLTableIterator() {} virtual void rewind() = 0; virtual bool next() = 0; virtual std::vector columnOffsets() const = 0; virtual const double* data() const = 0; virtual std::vector doublesDataSizes() const = 0; virtual std::vector columnsHaveMissing() const = 0; // n.b. don't use std::vector ... virtual std::vector missingValues() const = 0; }; using ColumnNames = std::vector; class SQLTable { public: SQLTable(SQLDatabase&, const std::string&, const std::string&); SQLTable(const SQLTable&) = delete; SQLTable& operator=(const SQLTable&) = delete; SQLTable(SQLTable&&) = delete; SQLTable& operator=(SQLTable&&) = delete; virtual ~SQLTable(); void loadIOMAP(std::istream&); void addColumn(const std::string& name, int index, const type::SQLType& type, bool hasMissingValue, double missingValue, bool isBitfield = false, const BitfieldDef& bitfieldDef = BitfieldDef()); void addLinkFrom(const SQLTable&); bool hasLinkFrom(const SQLTable&) const; void addLinkTo(const SQLTable&); bool hasLinkTo(const SQLTable&) const; bool isParentOf(const SQLTable&) const; virtual const SQLColumn& column(const std::string&) const; void updateColumnDoublesWidth(const std::string& name, size_t nDoubles); void updateColumnMissingValues(const std::string& name, bool hasMissing, double missingValue); // virtual SQLColumn& column(const std::string&); virtual bool hasColumn(const std::string& name) const; unsigned long long noRows() const; ColumnNames columnNames() const; FieldNames bitColumnNames(const std::string&) const; const std::string& name() const { return name_; } std::string fullName() const; SQLDatabase& owner() const { return owner_; } const std::string& path() const { return path_; } virtual void print(std::ostream& s) const; virtual SQLTableIterator* iterator(const std::vector>&, std::function metadataUpdateCallback) const = 0; protected: std::string path_; std::string name_; // std::map files_; std::map bitColumnNames_; std::map columnsByName_; std::map columnsByIndex_; std::vector> ownedColumns_; std::set> linksFrom_; std::set> linksTo_; // -- Methods void clearColumns(); // void print(std::ostream&) const; void addColumn(SQLColumn*, const std::string&, int); virtual SQLColumn* createSQLColumn(const type::SQLType& type, const std::string& name, size_t index, bool hasMissingValue, double missingValue, const BitfieldDef& d = BitfieldDef()); private: SQLDatabase& owner_; friend std::ostream& operator<<(std::ostream& s, const SQLTable& p) { p.print(s); return s; } }; // helper inline bool operator<(std::reference_wrapper lhs, std::reference_wrapper rhs) { return &lhs.get() < &rhs.get(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLParser.cc0000664000175000017500000001005015161702250017606 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/sql/SQLDatabase.h" #include "eckit/sql/SQLParser.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/SQLSession.h" #include "eckit/sql/SQLStatement.h" #include "eckit/sql/SchemaComponents.h" #include "eckit/sql/expression/BitColumnExpression.h" #include "eckit/sql/expression/ColumnExpression.h" #include "eckit/sql/expression/NumberExpression.h" #include "eckit/sql/expression/ParameterExpression.h" #include "eckit/sql/expression/SQLExpression.h" #include "eckit/sql/expression/SQLExpressions.h" #include "eckit/sql/expression/StringExpression.h" #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/type/SQLBitfield.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" #include "eckit/utils/StringTools.h" using namespace eckit; #define YYDEBUG 1 namespace SQLYacc { using eckit_sql_scan_t = void*; void eckit_sql_error(eckit_sql_scan_t, eckit::sql::SQLSession*, const char* msg); using namespace eckit; using namespace eckit::sql; using namespace eckit::sql::expression; using namespace eckit::sql::expression::function; #include "eckit/sql/sqly.c" void eckit_sql_error(eckit_sql_scan_t scanner, eckit::sql::SQLSession*, const char* msg) { std::stringstream os; struct eckit_sql_guts_t* eckit_sql_g = (struct eckit_sql_guts_t*)scanner; // internally we count the lines from 0... int lineNumber(1 + eckit_sql_g->eckit_sql_lineno_r); os << "SQL " << (msg ? msg : "syntax error") << ", line " << lineNumber // << " of " << yypath; << ". See https://software.ecmwf.int/wiki/display/ODBAPI/SQL\n"; throw SyntaxError(os.str()); } } // namespace SQLYacc SQLYacc::Stack& includeStack(void* eckit_sql_scanner) { SQLYacc::Stack* stack( static_cast(((struct SQLYacc::eckit_sql_guts_t*)eckit_sql_scanner)->eckit_sql_extra_r)); ASSERT(stack); return *stack; } extern "C" int eckit_sql_wrap(void* scanner) { return includeStack(scanner).pop(scanner); } namespace eckit::sql { // void SQLParser::parseString(SQLSession& session, const std::string& s, std::istream* is) //{ // // session.selectFactory().implicitFromTableSourceStream(is); // NOTIMP; //// session.selectFactory().config(cfg); //// session.selectFactory().csvDelimiter(csvDelimiter); // // parseStringInternal(session, s); // //// SQLYacc::eckit_sql_lex_init(&scanner); // TODO: handle unwind //} // // void SQLParser::parseString(SQLSession& session, const std::string& s, DataHandle* dh) //{ // // session.selectFactory().implicitFromTableSource(dh); // NOTIMP; //// session.selectFactory().config(cfg); // // parseStringInternal(session, s); //} // void SQLParser::parseString(SQLSession& session,const std::string& s, SQLDatabase& db, SQLOutputConfig cfg) //{ // session.selectFactory().database(&db); // session.selectFactory().config(cfg); // parseStringInternal(session, s); //} void SQLParser::parseString(SQLSession& session, const std::string& s) { SQLYacc::eckit_sql_scan_t scanner; SQLYacc::include_stack stack; SQLYacc::eckit_sql_lex_init_extra(&stack, &scanner); stack.push(cleanUpSQLText(s), "", (SQLYacc::YY_BUFFER_STATE)scanner, (SQLYacc::eckit_sql_scan_t)scanner); SQLYacc::eckit_sql_parse(scanner, &session); SQLYacc::eckit_sql_lex_destroy(scanner); } std::string SQLParser::cleanUpSQLText(const std::string& sql) { if (sql.empty()) { return sql; } std::string s(sql); s = eckit::StringTools::trim(s); s = eckit::StringTools::unQuote(s); s = eckit::StringTools::trim(s); if (!s.empty() && *s.rbegin() != ';') { s.push_back(';'); } return s; } } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLMATCHSubquerySession.cc0000664000175000017500000000247415161702250022325 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLMATCHSubquerySession.h" #include "eckit/sql/SQLMATCHSubquerySessionOutput.h" #include "odb_api/FunctionMATCH.h" namespace eckit { namespace sql { SQLMATCHSubquerySession::SQLMATCHSubquerySession(expression::function::FunctionMATCH& f) : SQLSession(odb::sql::SQLOutputConfig::defaultConfig(), ","), statement_(0), f_(f) { loadDefaultSchema(); } SQLMATCHSubquerySession::~SQLMATCHSubquerySession() {} SQLOutput* SQLMATCHSubquerySession::defaultOutput() { return new SQLMATCHSubquerySessionOutput(f_); } void SQLMATCHSubquerySession::statement(odb::sql::SQLStatement* sql) { ASSERT(sql); statement_ = sql; } SQLStatement* SQLMATCHSubquerySession::statement() { using P = odb::sql::SQLStatement*; if (gotSelectAST()) { gotSelectAST(false); statement_ = P(selectFactory().create(*this, selectAST())); } return statement_; } } // namespace sql } // namespace eckit eckit-2.0.7/src/eckit/sql/SelectOneTable.h0000664000175000017500000000350215161702250020471 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 2003 #ifndef eckit_sql_SelectOneTable_H #define eckit_sql_SelectOneTable_H #include "eckit/sql/expression/SQLExpressions.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLColumn; // Forward declarations class SQLTableIterator; struct SelectOneTable { SelectOneTable(const SQLTable* table = nullptr); ~SelectOneTable(); const SQLTable* table_; // Information about the data to be retrieved. std::vector> fetch_; std::vector*> values_; // std::vector> values_; // How do we find the data inside the allocated buffer in SQLSelect? // std::vector fetchSizeDoubles_; Expressions check_; Expressions index_; // For links std::pair offset_; std::pair length_; const SQLColumn* column_; // Reference column // For checking/debugging const SQLTable* table1_; const SQLTable* table2_; // For index // For sorting int order_; }; using SortedTables = std::vector; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SchemaAnalyzer.cc0000664000175000017500000002432515161702250020712 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SchemaAnalyzer.h" #include "eckit/exception/Exceptions.h" #include "eckit/utils/StringTools.h" using namespace eckit; namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SchemaAnalyzer::SchemaAnalyzer() {} SchemaAnalyzer::~SchemaAnalyzer() {} void SchemaAnalyzer::addTable(TableDef& table) { tableDefs_.push_back(table); for (auto& col : table.columns()) { std::string fullname = col.name() + "@" + table.name(); columnTypes_[fullname] = col.type(); // Log::debug() << "SchemaAnalyzer::addTable(): columnTypes_[" << fullname << "] = " // << columnTypes_[fullname] << std::endl; } } void SchemaAnalyzer::addBitfieldType(const std::string& name, const FieldNames& fields, const Sizes& sizes) { bitfieldTypes_[name] = make_pair(fields, sizes); } const BitfieldDef& SchemaAnalyzer::getBitfieldType(const std::string& typeName) const { auto it = bitfieldTypes_.find(typeName); if (it != bitfieldTypes_.end()) { return it->second; } static const BitfieldDef nullBitfieldDef; return nullBitfieldDef; } const BitfieldDef& SchemaAnalyzer::getBitfieldTypeDefinition(const std::string& columnName) const { auto it = columnTypes_.find(columnName); ASSERT(it != columnTypes_.end()); return getBitfieldType(it->second); } std::string SchemaAnalyzer::generateSelectAll(const std::set& skipTables) const { std::string from; std::string selectlist; if (tableDefs_.empty()) { return ""; } for (const auto& table : tableDefs_) { // Tables to skip? if (skipTables.find(table.name()) != skipTables.end()) { continue; } if (!from.empty()) { from += ", "; } from += table.name(); for (const auto& col : table.columns()) { std::string fullname = col.name() + "@" + table.name(); if (col.type() == "@LINK") { Log::info() << "SchemaAnalyzer::generateSelectAll(): Skipping " << fullname << std::endl; } else { if (!selectlist.empty()) { selectlist += ", "; } selectlist += fullname; } } selectlist += "\n"; } return "\nSELECT\n" + selectlist + "\n FROM\n" + from; } TableDefs SchemaAnalyzer::definitions() const { return tableDefs_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #if 0 SchemaAnalyzer::SchemaAnalyzer() { //desc, hdr, sat, reo3, satob, rtovs, rtovs_slev, rtovs_mlev, rtovs_pred, atovs, atovs_pred, scatt, ssmi, ssmi_slev, ssmi_mlev, body, errstat, update, rtovs_body, ssmi_body, scatt_body skipTable("poolmask"); skipTable("bufr"); skipTable("ddrs"); skipTable("index"); skipTable("rtovs"); skipTable("rtovs_slev"); skipTable("rtovs_mlev"); skipTable("rtovs_pred"); skipTable("rtovs_body"); skipTable("ssmi"); skipTable("ssmi_body"); skipTable("ssmi_slev"); skipTable("ssmi_pred"); skipTable("ssmi_mlev"); skipTable("atovs"); skipTable("atovs_pred"); skipTable("scatt"); skipTable("scatt_body"); skipTable("reo3"); skipTable("sat"); skipTable("satob"); skipTable("update"); skipTable(""); skipTable(""); } SchemaAnalyzer::~SchemaAnalyzer() {} void SchemaAnalyzer::beginSchema(const std::string& name) { if (!currentSchema_.empty()) { std::string message = "Cannot create new schema '" + name + "' - current schema '" + currentSchema_ + "' not finalized"; throw eckit::UserError(message); } std::pair result; result = schemas_.insert(make_pair(name, SchemaDef())); if (result.second == false) { std::string message = "Schema '" + name + "' already defined"; throw eckit::UserError(message); } currentSchema_ = name; } void SchemaAnalyzer::endSchema() { currentSchema_.clear(); } void SchemaAnalyzer::addTable(TableDef& table) { std::string schemaName = ""; if (StringTool::isInQuotes(table.name())) table.name(StringTool::unQuote(table.name())); else { size_t pos = table.name().find("."); if (pos != std::string::npos) { schemaName = table.name().substr(0, pos); table.name(table.name().substr(pos + 1)); } } ColumnDefs& columns (table.columns()); for (ColumnDefs::iterator it = columns.begin(); it != columns.end(); ++it) { ColumnDef& column(*it); column.name(column.name() + "@" + table.name()); columnTypes_[column.name()] = column.type(); if (isBitfield(column.name())) column.bitfieldDef(getBitfieldTypeDefinition(column.name())); } for (int i = 0, n = table.parents().size(); i < n; i++) { TableDefs::const_iterator it(tableDefs_.find(table.parents()[i])); if (it == tableDefs_.end()) throw eckit::UserError(std::string("Could not find definition of parent table '") + table.parents()[i] + "' inherited by table '" + table.name() + "'"); const TableDef& parent (it->second); if(! parent.parents().empty()) throw UserError("More than 1-level inheritance not supported"); for (ColumnDefs::const_iterator c (parent.columns().begin()); c != parent.columns().end(); ++c) table.columns().push_back(*c); } if (currentSchema_.empty() && schemaName.empty()) { std::pair result (tableDefs_.insert(std::pair(table.name(), table))); if (result.second == false) throw eckit::UserError(std::string ("Table '") + table.name() + "' already defined"); } else { if (schemaName.empty()) schemaName = currentSchema_; SchemaDefs::iterator it (schemas_.find(schemaName)); if (it == schemas_.end()) throw eckit::UserError(std::string("Referenced schema '") + schemaName + "' not defined '"); SchemaDef& schema (it->second); TableDefs& tables (schema.tables()); std::pair result (tables.insert(std::pair(table.name(), table))); if (result.second == false) throw eckit::UserError(std::string ("Table '") + table.name() + "' already defined in '" + schemaName + "' schema"); } } void SchemaAnalyzer::skipTable(std::string tableName) { tablesToSkip_.insert(tableName); } std::string SchemaAnalyzer::generateSELECT() const { std::string from = ""; std::string selectList = ""; if (tableDefs_.size() == 0) return ""; for (TableDefs::const_iterator t = tableDefs_.begin(); t != tableDefs_.end(); ++t) { TableDef tableDef = t->second; std::string tableName = tableDef.name(); if (tablesToSkip_.find(tableName) != tablesToSkip_.end()) continue; from += tableName + ", "; ColumnDefs columnDefs = tableDef.columns(); for (ColumnDefs::const_iterator i = columnDefs.begin(); i != columnDefs.end(); i++) { const std::string typeName = i->type(); if (typeName == "@LINK") { Log::info() << "SchemaAnalyzer::generateSELECT: Skipping " << i->name() << std::endl; continue; } selectList += i->name() + ", "; } selectList += "\n"; } return "\nSELECT\n" + selectList + "\n FROM\n" + from; } Definitions SchemaAnalyzer::generateDefinitions() { return Definitions(schemas_, tableDefs_); } void SchemaAnalyzer::addBitfieldType(const std::string& name, const FieldNames& fields, const Sizes& sizes, const std::string& typeSignature) { bitfieldTypes_[name] = make_pair(fields, sizes); } bool SchemaAnalyzer::isBitfield(const std::string& columnName) const { for (std::map::const_iterator it(columnTypes_.begin()); it != columnTypes_.end(); ++it) { if (it->first == columnName) { std::string columnType (it->second); for (std::map::const_iterator it(bitfieldTypes_.begin()); it != bitfieldTypes_.end(); ++it) if (it->first == columnType) return true; } } return false; } const BitfieldDef& SchemaAnalyzer::getBitfieldTypeDefinition(const std::string& columnName) { ASSERT(isBitfield(columnName)); std::string columnType = columnTypes_.find(columnName)->second; return bitfieldTypes_[columnType]; } void SchemaAnalyzer::updateBitfieldsDefs(MetaData &md, std::map & truenames) const { for (size_t i = 0; i < md.size(); i++) { Column &c (*md[i]); if (c.type() == BITFIELD) c.bitfieldDef(const_cast(this)->getBitfieldTypeDefinition(truenames[c.name()])); } } bool SchemaAnalyzer::tableKnown(const std::string& name) const { return tableDefs_.find(name) != tableDefs_.end(); } const TableDef& SchemaAnalyzer::findTable(const std::string& name) const { for (TableDefs::const_iterator it(tableDefs_.begin()); it != tableDefs_.end(); ++it) if (it->first == name) return it->second; throw eckit::UserError(std::string("Table '" + name + "' not found")); } std::string SchemaAnalyzer::findColumnType(const std::string& columnName) { std::vector ps (StringTools::split("@", columnName)); if (ps.size() != 2) throw eckit::UserError(std::string("Expected fully qualified column name (@), got: " + columnName)); std::string name (ps[0]), table (ps[1]); const TableDef& tableDef (findTable(table)); ColumnDefs columnDefs (tableDef.columns()); for (ColumnDefs::const_iterator it (columnDefs.begin()); it != columnDefs.end(); ++it) if (it->name() == columnName) return it->type(); throw eckit::UserError(std::string("Column " + columnName + " not found")); } #endif eckit-2.0.7/src/eckit/sql/SQLSession.cc0000664000175000017500000001236415161702250020007 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/FileHandle.h" #include "eckit/log/Timer.h" #include "eckit/utils/StringTools.h" #include "eckit/sql/SQLDatabase.h" #include "eckit/sql/SQLOutput.h" #include "eckit/sql/SQLOutputConfig.h" #include "eckit/sql/SQLParser.h" #include "eckit/sql/SQLSession.h" #include "eckit/sql/SQLStatement.h" #include "eckit/sql/SQLTableFactory.h" using namespace eckit; namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SQLSession::SQLSession(std::unique_ptr out, std::unique_ptr config, const std::string& csvDelimiter) : selectFactory_(*this), lastExecuteResult_(), config_(config ? std::move(config) : std::unique_ptr(new SQLOutputConfig())), output_(std::move(out)), csvDelimiter_(csvDelimiter) { ASSERT(output_ || config_); database_.open(); } SQLSession::SQLSession(std::unique_ptr config, const std::string& csvDelimiter) : SQLSession(nullptr, std::move(config), csvDelimiter) {} SQLSession::SQLSession(std::unique_ptr out, const std::string& csvDelimiter) : SQLSession(std::move(out), nullptr, csvDelimiter) {} SQLSession::SQLSession(const std::string& csvDelimiter) : SQLSession(nullptr, nullptr, csvDelimiter) {} SQLSession::~SQLSession() {} SQLSelectFactory& SQLSession::selectFactory() { return selectFactory_; } // SQLInsertFactory& SQLSession::insertFactory() { return insertFactory_; } unsigned long long SQLSession::execute(SQLStatement& sql) { Timer timer("Execute"); unsigned long long n = sql.execute(); return lastExecuteResult_ = n; } std::unique_ptr SQLSession::newFileOutput(const eckit::PathName& path) { return std::unique_ptr(config_->buildOutput(path)); } const SQLDatabase& SQLSession::currentDatabase() const { return database_; } SQLDatabase& SQLSession::currentDatabase() { return database_; } void SQLSession::setStatement(SQLStatement* s) { statement_.reset(s); } SQLStatement& SQLSession::statement() { ASSERT(statement_); return *statement_; } SQLOutput& SQLSession::output() { ASSERT(output_ || config_); if (!output_) { output_.reset(config_->buildOutput()); } return *output_; } SQLTable& SQLSession::findTable(const std::string& name) { if (!currentDatabase().hasTable(name)) { Log::debug() << "No table named \"" << name << "\" found. Looking in table factory" << std::endl; currentDatabase().addTable(SQLTableFactory::instance().build(currentDatabase(), name)); } return currentDatabase().table(name); } /* SQLTable* SQLSession::openDataStream(std::istream &is, const std::string& delimiter) { return currentDatabase().openDataStream(is, delimiter); } SQLTable* SQLSession::openDataHandle(DataHandle &dh) { return currentDatabase().openDataHandle(dh); } */ void SQLSession::loadDefaultSchema() { std::string schemaPathName(schemaFile()); if (schemaPathName.empty()) { return; } Log::debug() << "Loading schema " << schemaPathName << std::endl; FileHandle dh(schemaPathName); size_t sz = dh.openForRead(); std::string schema(sz, ' '); ASSERT(size_t(dh.read(&schema[0], sz)) == sz); SQLParser parser; // TODO: update include path parser.parseString(*this, schema); } std::string SQLSession::schemaFile() { // TODO; This resource is a really bad way of doing this... static std::string pathName(eckit::StringTools::unQuote(Resource("$ECKIT_SQL_SCHEMA_PATH", ""))); return pathName; } std::string SQLSession::readIncludeFile(const std::string& fileName) { std::vector dirs(includePaths()); Log::debug() << "read include: " << fileName << std::endl; for (size_t i(0); i < dirs.size(); ++i) { PathName pathName(dirs[i] + fileName); Log::debug() << "Looking for include file " << fileName << " in " << dirs[i] << std::endl; if (!pathName.exists()) { continue; } FileHandle dh(pathName); size_t sz = dh.openForRead(); std::string readBuf(sz, ' '); ASSERT(size_t(dh.read(&readBuf[0], sz)) == sz); return readBuf; } throw eckit::UserError(std::string("Include file '") + fileName + "' not found"); } std::vector SQLSession::includePaths() { std::vector r; r.push_back(""); r.push_back(PathName(schemaFile()).dirName().asString() + "/"); r.push_back("./"); return r; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLColumn.h0000664000175000017500000000557515161702250017471 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLColumn.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLColumn_H #define SQLColumn_H namespace eckit { class PathName; } #include "eckit/sql/SQLIterator.h" #include "eckit/sql/SQLTypedefs.h" namespace eckit::sql { class SQLBitColumn; class SQLTable; class SQLColumn : public SQLIterator { public: SQLColumn(const type::SQLType&, SQLTable&, const std::string&, size_t index, bool hasMissingValue, double missingValue, const BitfieldDef& d = BitfieldDef()); ~SQLColumn() override; void scan(); unsigned long long noRows() const; const std::string& name() const { return name_; } int index() const { return index_; } void index(int i) { index_ = i; } std::string fullName() const; SQLTable* table() const; bool hasMissingValue() const { return hasMissingValue_; } double missingValue() const { return missingValue_; } void hasMissingValue(bool flag) { hasMissingValue_ = flag; } void missingValue(double val) { missingValue_ = val; } bool isBitfield() const { return isBitfield_; } bool isMissingValue(const double* val) const { // return hasMissingValue_ && (*val == missingValue_); const char* punnable_val = reinterpret_cast(val); const char* punnable_missing = reinterpret_cast(&missingValue_); const uint64_t* punned_val = reinterpret_cast(punnable_val); const uint64_t* punned_missing = reinterpret_cast(punnable_missing); return hasMissingValue_ && (*punned_val == *punned_missing); } const BitfieldDef& bitfieldDef() const { return bitfieldDef_; } size_t dataSizeDoubles() const override { return sizeDoubles_; } // -- Overridden methods // From SQLIterator void rewind() override; double next(bool& missing) override; void advance(unsigned long) override; protected: unsigned long long noRows_; void print(std::ostream&) const override; // private: protected: SQLColumn(const SQLColumn&); SQLColumn& operator=(const SQLColumn&); void setPool(int); SQLTable& owner_; std::string name_; int index_; std::vector rows_; std::vector iterators_; long long current_; long long last_; long long position_; SQLIterator* iterator_; bool hasMissingValue_; double missingValue_; bool isBitfield_; const BitfieldDef bitfieldDef_; size_t sizeDoubles_; }; } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/expression/0000775000175000017500000000000015161702250017666 5ustar alastairalastaireckit-2.0.7/src/eckit/sql/expression/ColumnExpression.cc0000664000175000017500000001563615161702250023525 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/ColumnExpression.h" #include #include #include "eckit/sql/SQLColumn.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/SQLTable.h" #include "eckit/sql/expression/ShiftedColumnExpression.h" #include "eckit/utils/Translator.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- ColumnExpression::ColumnExpression(const std::string& name, const SQLTable* table, int begin, int end) : type_{nullptr}, value_{nullptr}, columnName_(name), table_(table), tableReference_(), beginIndex_(begin), endIndex_(end), nominalShift_(0) {} ColumnExpression::ColumnExpression(const std::string& name, const std::string& tableReference, int begin, int end) : type_{nullptr}, value_{nullptr}, columnName_(name), table_{nullptr}, tableReference_(tableReference), beginIndex_(begin), endIndex_(end), nominalShift_(0) {} std::shared_ptr ColumnExpression::clone() const { return std::make_shared(*this); } std::shared_ptr ColumnExpression::reshift(int minColumnShift) const { return std::make_shared>(*this, -minColumnShift, 0); } ColumnExpression::~ColumnExpression() {} double ColumnExpression::eval(bool& missing) const { // n.b. we should only ever get here for numerical columns. Probably should ASSERT() if (value_->second) { missing = true; } return *(value_->first); } void ColumnExpression::eval(double* out, bool& missing) const { if (value_->second) { missing = true; } ::memcpy(out, value_->first, type_->size()); } std::string ColumnExpression::evalAsString(bool& missing) const { if (value_->second) { missing = true; } return type_->asString(value_->first); } void ColumnExpression::preprepare(SQLSelect& sql) { /// pre-prepare exists to determine the Table/SQL column combination that is needed. /// /// This is then used in the SQLSelect to determine buffer sizes and data offsets, which /// can then be used to initialise value_ and name_ in the prepare() function. /// /// There is no straightforward way to do these both in one prepare() statement, without /// shifting the functionality outside. if (!table_) { table_ = &sql.findTable(columnName_); } sql.ensureFetch(*table_, columnName_); // Get the details into the ColumnExpression const SQLColumn& column(table_->column(columnName_)); fullName_ = tableColumnToFullname(column); hasMissingValue_ = column.hasMissingValue(); missingValue_ = column.missingValue(); isBitfield_ = column.isBitfield(); bitfieldDef_ = column.bitfieldDef(); if (columnName_ == title() && columnName_ != fullName_) { title(fullName_); Log::debug() << "ColumnExpression::preprepare: columnName_=" << columnName_ << ", title[PATCHED]=" << title() << std::endl; } } std::string ColumnExpression::tableColumnToFullname(const SQLColumn& column) const { return column.fullName(); } void ColumnExpression::prepare(SQLSelect& sql) { updateType(sql); } void ColumnExpression::updateType(SQLSelect& sql) { // Get the memory address associated with reading from the column. (this is _not_ // the SQLColumn object used in preprepare (which describe the columns requested // in the Select statement), but instead describes the mapping to the // SQLTableIterator object -- which describes what/how we are actually getting the // data. value_ = &sql.column(columnName_, table_); type_ = sql.typeOf(columnName_, table_); Log::debug() << "ColumnExpression::updateType: columnName_=" << columnName_ << ", title=" << title() << ", table=" << table_->name() << ", fullName =" << fullName_ << " type=" << *type_ << " bitfieldDef.first.size =" << bitfieldDef_.first.size() << std::endl; } void ColumnExpression::cleanup(SQLSelect& sql) { value_ = nullptr; type_ = nullptr; } void ColumnExpression::print(std::ostream& s) const { s << columnName_; if (nominalShift_ != 0) { s << "#" << nominalShift_; } // if(table_) s << "@" << table_->fullName(); } void ColumnExpression::output(SQLOutput& o) const { type_->output(o, value_->first, value_->second); } void ColumnExpression::expandStars(const std::vector>& tables, expression::Expressions& e) { std::ostream& L(Log::debug()); L << "ColumnExpression::expandStars: expanding '" << columnName_ << "' (" << tableReference_ << ")" << std::endl; if (beginIndex_ != -1 && endIndex_ != -1) { ASSERT(beginIndex_ <= endIndex_); for (int i = beginIndex_; i <= endIndex_; i++) { e.push_back(std::make_shared(columnName_ + "_" + Translator()(i), this->table())); } return; } if (columnName_ != "*") { // replace ColumnExpressions referring to aliases created with AS with references to appropriate expressions // If it's not an alias then: e.push_back(shared_from_this()); return; } std::stringstream ss; unsigned int matched = 0; for (const SQLTable& table : tables) { std::vector names = table.columnNames(); for (size_t i = 0; i < names.size(); i++) { if ((tableReference_.size()) && ((names[i].rfind(tableReference_) == std::string::npos) || (names[i].rfind(tableReference_) + tableReference_.size() < names[i].size()))) { L << "ColumnExpression::expandStars: skip '" << names[i] << "'" << std::endl; continue; } ss << names[i] << ", "; ++matched; e.push_back(std::make_shared(names[i], &table)); } } if (!matched) { throw eckit::UserError(std::string("No columns matching ") + columnName_ + tableReference_ + " found."); } L << "ColumnExpression::expandStars: added " << ss.str() << std::endl; } void ColumnExpression::tables(std::set& t) { ASSERT(table_); t.insert(table_); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/SQLExpressionEvaluated.cc0000664000175000017500000000567515161702250024564 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/SQLExpressionEvaluated.h" #include #include "eckit/exception/Exceptions.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- SQLExpressionEvaluated::SQLExpressionEvaluated(SQLExpression& e) : type_(e.type()), missing_(false), missingValue_(e.missingValue()) { size_t byteSize = type_->size(); ASSERT(byteSize % sizeof(double) == 0); value_.resize(byteSize / sizeof(double)); e.eval(&value_[0], missing_); hasMissingValue_ = e.hasMissingValue(); } SQLExpressionEvaluated::~SQLExpressionEvaluated() {} void SQLExpressionEvaluated::print(std::ostream& o) const { if (missing_) { o << "NULL"; } else { o << type_->asString(&value_[0]); } o << ", "; } double SQLExpressionEvaluated::eval(bool& missing) const { if (missing_) { missing = true; } return value_[0]; } void SQLExpressionEvaluated::eval(double* out, bool& missing) const { if (missing_) { missing = true; } ::memcpy(out, &value_[0], value_.size() * sizeof(value_[0])); } std::string SQLExpressionEvaluated::evalAsString(bool& missing) const { if (missing_) { missing = true; } else { return type_->asString(&value_[0]); } return std::string(); } void SQLExpressionEvaluated::output(SQLOutput& o) const { type_->output(o, &value_[0], missing_); } void SQLExpressionEvaluated::prepare(SQLSelect&) { NOTIMP; } void SQLExpressionEvaluated::updateType(SQLSelect&) { NOTIMP; } void SQLExpressionEvaluated::cleanup(SQLSelect&) { NOTIMP; } bool SQLExpressionEvaluated::isAggregate() const { NOTIMP; } bool SQLExpressionEvaluated::isConstant() const { return true; } bool SQLExpressionEvaluated::isNumber() const { return type_->getKind() != type::SQLType::stringType; } std::shared_ptr SQLExpressionEvaluated::simplify(bool&) { return nullptr; } std::shared_ptr SQLExpressionEvaluated::clone() const { return std::make_shared(*this); } std::shared_ptr SQLExpressionEvaluated::reshift(int minColumnShift) const { throw eckit::SeriousBug("Attempting to row-shift already-evaluated expression", Here()); } const type::SQLType* SQLExpressionEvaluated::type() const { return type_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/ConstantExpression.cc0000664000175000017500000000270115161702250024046 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/ConstantExpression.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- ConstantExpression::ConstantExpression(double v, bool missing, const type::SQLType* type) : value_(v), missing_(missing), type_(*type) {} ConstantExpression::ConstantExpression(const ConstantExpression& rhs, const ConstantExpression::PrivateKey&) : isBitfield_(rhs.isBitfield_), bitfieldDef_(rhs.bitfieldDef_), hasMissingValue_(rhs.hasMissingValue_), missingValue_(rhs.missingValue_), value_(rhs.value_), missing_(rhs.missing_), type_(rhs.type_) {} ConstantExpression::~ConstantExpression() {} void ConstantExpression::output(SQLOutput& o) const { type_.output(o, value_, missing_); } const type::SQLType* ConstantExpression::type() const { return &type_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/BitColumnExpression.cc0000664000175000017500000001172615161702250024160 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/BitColumnExpression.h" #include #include "eckit/filesystem/PathName.h" #include "eckit/os/BackTrace.h" #include "eckit/sql/SQLColumn.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/SQLTable.h" #include "eckit/sql/expression/ShiftedColumnExpression.h" #include "eckit/sql/type/SQLBit.h" #include "eckit/utils/Tokenizer.h" // Cray C++ compiler should not try to optimize this code #if _CRAYC #pragma _CRI noopt #endif namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- BitColumnExpression::BitColumnExpression(const std::string& name, const std::string& field, SQLTable* table) : ColumnExpression(name + "." + field + "@" + table->name(), table), mask_(0), bitShift_(0), field_(field), name_(name) { Log::debug() << "BitColumnExpression::BitColumnExpression: name=" << name << ", field=" << field << ", table->name() =" << table->name() << ": name_=" << name_ << std::endl; } BitColumnExpression::BitColumnExpression(const std::string& name, const std::string& field, const std::string& tableReference) : ColumnExpression(name + "." + field + tableReference, tableReference), mask_(0), bitShift_(0), field_(field), name_(name) { Log::debug() << "BitColumnExpression::BitColumnExpression: name=" << name << ", field=" << field << ", tableReference=" << tableReference << ": name_=" << name_ << std::endl; } BitColumnExpression::BitColumnExpression(const BitColumnExpression& o) : ColumnExpression(o), mask_(o.mask_), bitShift_(o.bitShift_), field_(o.field_), name_(o.name_) {} BitColumnExpression::~BitColumnExpression() {} const eckit::sql::type::SQLType* BitColumnExpression::type() const { // Change the type to integer to be able to create a new ODA if necessary return &eckit::sql::type::SQLType::lookup("integer"); } void BitColumnExpression::prepare(SQLSelect& sql) { updateType(sql); } void BitColumnExpression::updateType(SQLSelect& sql) { std::string name = name_ + "." + field_ + tableReference_; if (!table_) { table_ = &sql.findTable(name); } value_ = &sql.column(name, table_); type_ = sql.typeOf(name, table_); const type::SQLBit* bit = dynamic_cast(type_); if (bit) { mask_ = bit->mask(); bitShift_ = bit->shift(); } else { // This is for .length and .offset // Not very nice, I know mask_ = 0xffffffff; bitShift_ = 0; } } double BitColumnExpression::eval(bool& missing) const { if (value_->second) { missing = true; } unsigned long x = static_cast(*value_->first); return (x & mask_) >> bitShift_; } void BitColumnExpression::expandStars(const std::vector>& tables, expression::Expressions& e) { using namespace eckit; using namespace std; // TODO: regex if (field_ != "*") { e.push_back(shared_from_this()); return; } for (const SQLTable& table : tables) { std::vector names = table.bitColumnNames(name_ + tableReference_); for (size_t i = 0; i < names.size(); i++) { e.push_back(std::make_shared(name_, names[i], tableReference_ /*table*/)); } } } std::shared_ptr BitColumnExpression::clone() const { return std::make_shared(*this); } std::shared_ptr BitColumnExpression::reshift(int minColumnShift) const { return std::make_shared>(*this, -minColumnShift, 0); } std::string BitColumnExpression::tableColumnToFullname(const SQLColumn& column) const { // The fully expanded table column will not include the field name, as we are extracting // the specified bits out of that column. // // This means that to construct the fully qualified field name, we need to include // these field names std::vector bits; Tokenizer('@')(column.fullName(), bits); ASSERT(bits.size() > 0); ASSERT(bits.size() < 3); if (bits[0] != name_ + "." + field_) { bits[0] += "." + field_; } if (bits.size() == 2) { return bits[0] + "@" + bits[1]; } return bits[0]; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/ShiftedColumnExpression.h0000664000175000017500000000532215161702250024665 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// ECMWF Dec 2012 #ifndef eckit_sql_ShiftedColumnExpression_H #define eckit_sql_ShiftedColumnExpression_H #include #include namespace eckit { namespace sql { class SQLOutput; class SQLTable; class SQLSelect; namespace expression { //---------------------------------------------------------------------------------------------------------------------- template class ShiftedColumnExpression : public T { public: ShiftedColumnExpression(const std::string&, SQLTable*, int shift, int nominalShift, int begin = -1, int end = -1); ShiftedColumnExpression(const std::string&, const std::string& tableReference, int shift, int nominalShift, int begin = -1, int end = -1); ShiftedColumnExpression(const ShiftedColumnExpression&); ShiftedColumnExpression(const T& o, int shift, int nominalShift); // for bitfields columns ShiftedColumnExpression(const std::string& name, const std::string& field, SQLTable* table, int shift, int nominalShift); ShiftedColumnExpression(const std::string& name, const std::string& field, const std::string& tableReference, int shift, int nominalShift); ~ShiftedColumnExpression(); SQLTable* table(); double* current(); std::shared_ptr clone() const override; std::shared_ptr reshift(int minColumnShift) const override; int shift() const { return shift_; } int nominalShift() const { return nominalShift_; } protected: int shift_; // For the HASH operator int nominalShift_; // For the HASH operator // -- Overridden methods void print(std::ostream& s) const override; void cleanup(SQLSelect& sql) override; using T::eval; double eval(bool& missing) const override; void output(SQLOutput& s) const override; private: ShiftedColumnExpression& operator=(const ShiftedColumnExpression&); void allocateCircularBuffer(); std::list > oldValues_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace expression } // namespace sql } // namespace eckit #include "eckit/sql/expression/ShiftedColumnExpression.cc" #endif eckit-2.0.7/src/eckit/sql/expression/StringExpression.cc0000664000175000017500000001045015161702250023523 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/StringExpression.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLTable.h" #include "eckit/sql/expression/ColumnExpression.h" #include "eckit/sql/expression/SQLExpressions.h" #include "eckit/utils/Regex.h" //---------------------------------------------------------------------------------------------------------------------- namespace { bool isColumnRegex(const std::string& s) { return (s[0] == '/' && s[s.size() - 1] == '/') || (s[0] == '~' && s[1] == '/' && s[s.size() - 1] == '/'); } bool matchEx(const std::string& regex, const std::string& s) { bool negated = false; std::string rx = regex; if (rx[0] == '~') { rx.erase(0, 0); negated = true; } // TODO: remove '/' bool matches = eckit::Regex(rx).match(s); return (negated && !matches) || (!negated && matches); } } // namespace namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- const eckit::sql::type::SQLType* StringExpression::type() const { return type_; } StringExpression::StringExpression(const std::string& name) : name_(name) { size_t len = name.length(); size_t lenDoubles = (len == 0) ? 1 : ((len - 1) / sizeof(double)) + 1; size_t lenChars = lenDoubles * sizeof(double); value_.resize(lenDoubles); char* val = reinterpret_cast(&value_[0]); ::memcpy(val, name.c_str(), len); if (len != lenChars) { ::memset(val + len, 0, lenChars - len); } type_ = &type::SQLType::lookup("string", lenDoubles); } StringExpression::StringExpression(const StringExpression& o) : name_(o.name_), value_(o.value_) {} void StringExpression::expandStars(const std::vector>& tables, expression::Expressions& e) { std::ostream& L(Log::info()); // TODO: if(verbose_) {...} // Log::info() << "StringExpression::expandStars: name_: '" << name_ << "', value_: '" << value_ << "'" << // std::endl; if (!isColumnRegex(name_)) { e.push_back(shared_from_this()); return; } unsigned int matched = 0; for (const auto& table : tables) { std::vector names = table.get().columnNames(); for (size_t i = 0; i < names.size(); i++) { const std::string& name = names[i]; if (!matchEx(name_, name)) { L << "StringExpression::expandStars: skip '" << name << "'" << std::endl; continue; } L << "StringExpression::expandStars: adding '" << name << "'" << std::endl; ++matched; e.push_back(std::make_shared(name, &table.get())); } } if (!matched) { throw eckit::UserError(std::string("No columns matching regex '") + name_ + "' found."); } } std::shared_ptr StringExpression::clone() const { return std::make_shared(name_); } std::shared_ptr StringExpression::reshift(int minColumnShift) const { return clone(); } StringExpression::~StringExpression() {} double StringExpression::eval(bool& missing) const { return value_[0]; } void StringExpression::eval(double* out, bool& missing) const { ::memcpy(out, &value_[0], value_.size() * sizeof(value_[0])); } std::string StringExpression::evalAsString(bool& missing) const { return name_; } void StringExpression::prepare(SQLSelect& sql) {} void StringExpression::cleanup(SQLSelect& sql) {} void StringExpression::output(SQLOutput& o) const { type_->output(o, &value_[0], false); } void StringExpression::print(std::ostream& s) const { s << "'" << name_ << "'"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/NumberExpression.cc0000664000175000017500000000310715161702250023506 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/NumberExpression.h" #include namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- NumberExpression::NumberExpression(double value) : SQLExpression(), value_(value) {} NumberExpression::NumberExpression(const NumberExpression& other) : SQLExpression(), value_(other.value_) {} std::shared_ptr NumberExpression::clone() const { return std::make_shared(*this); } std::shared_ptr NumberExpression::reshift(int minColumnShift) const { return clone(); } NumberExpression::~NumberExpression() {} const type::SQLType* NumberExpression::type() const { return &type::SQLType::lookup("real"); } double NumberExpression::eval(bool& missing) const { return value_; } void NumberExpression::prepare(SQLSelect& sql) {} void NumberExpression::cleanup(SQLSelect& sql) {} void NumberExpression::print(std::ostream& s) const { s << value_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/ShiftedColumnExpression.cc0000664000175000017500000001167315161702250025031 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" namespace eckit { namespace sql { namespace expression { //---------------------------------------------------------------------------------------------------------------------- template void ShiftedColumnExpression::allocateCircularBuffer() { // Log::info() << "allocateCircularBuffer:" << shift_ << std::endl; // FIXME: we need to retrieve actual value of missing value for this column double const MISSING_VALUE_REAL = -2147483647.0; static std::pair missing_(MISSING_VALUE_REAL, true); ASSERT(shift_ > 0); for (size_t i = 0; i < size_t(shift_); ++i) oldValues_.push_front(missing_); } template ShiftedColumnExpression::ShiftedColumnExpression(const T& o, int shift, int nominalShift) : T(o), shift_(shift), nominalShift_(nominalShift), oldValues_() {} template ShiftedColumnExpression::ShiftedColumnExpression(const std::string& name, SQLTable* table, int shift, int nominalShift, int begin, int end) : T(name, table, begin, end), shift_(shift), nominalShift_(nominalShift), oldValues_() {} template ShiftedColumnExpression::ShiftedColumnExpression(const std::string& name, const std::string& tableReference, int shift, int nominalShift, int begin, int end) : T(name, tableReference, begin, end), shift_(shift), nominalShift_(nominalShift), oldValues_() {} template ShiftedColumnExpression::ShiftedColumnExpression(const std::string& name, const std::string& field, SQLTable* table, int shift, int nominalShift) : T(name, field, table), shift_(shift), nominalShift_(nominalShift), oldValues_() {} template ShiftedColumnExpression::ShiftedColumnExpression(const std::string& name, const std::string& field, const std::string& tableReference, int shift, int nominalShift) : T(name, field, tableReference), shift_(shift), nominalShift_(nominalShift), oldValues_() {} template ShiftedColumnExpression::ShiftedColumnExpression(const ShiftedColumnExpression& e) : T(e), shift_(e.shift_), nominalShift_(e.nominalShift_), oldValues_(e.oldValues_) {} template std::shared_ptr ShiftedColumnExpression::clone() const { return std::make_shared>(*this); } template std::shared_ptr ShiftedColumnExpression::reshift(int minColumnShift) const { int newshift = shift() - minColumnShift; ASSERT(newshift >= 0); if (newshift == 0) { auto r = std::make_shared(*this); r->nominalShift(nominalShift()); return r; } else { return std::make_shared>(*this, newshift, nominalShift()); } } template ShiftedColumnExpression::~ShiftedColumnExpression() {} template void ShiftedColumnExpression::print(std::ostream& s) const { s << this->columnName_; if (nominalShift_ != 0) s << "#" << nominalShift_; } template double ShiftedColumnExpression::eval(bool& missing) const { ShiftedColumnExpression& self(*const_cast*>(this)); ASSERT(shift_ > 0); if (oldValues_.size() == 0) self.allocateCircularBuffer(); std::pair const& v(self.oldValues_.back()); double value = v.first; bool miss = v.second; self.oldValues_.pop_back(); std::pair ev; ev.first = T::eval(ev.second); self.oldValues_.push_front(ev); if (miss) missing = true; return value; } template void ShiftedColumnExpression::cleanup(SQLSelect& sql) { this->value_ = 0; this->type_ = 0; oldValues_.clear(); } template void ShiftedColumnExpression::output(SQLOutput& o) const { // Log::info() << "ShiftedColumnExpression::output:" << std::endl; bool missing = false; double v = eval(missing); this->type_->output(o, v, missing); } template SQLTable* ShiftedColumnExpression::table() { return this->table_; } template double* ShiftedColumnExpression::current() { NOTIMP; return &(this->value_->first); } //---------------------------------------------------------------------------------------------------------------------- } // namespace expression } // namespace sql } // namespace eckit eckit-2.0.7/src/eckit/sql/expression/ConstantExpression.h0000664000175000017500000000602015161702250023706 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// ECMWF Oct 11 #ifndef eckit_sql_ConstantExpression_H #define eckit_sql_ConstantExpression_H #include "eckit/exception/Exceptions.h" #include "eckit/sql/expression/SQLExpression.h" namespace eckit::sql { class SQLSelect; class SQLTable; class SQLOutput; namespace expression { //---------------------------------------------------------------------------------------------------------------------- class ConstantExpression : public SQLExpression { struct PrivateKey {}; // This is just to allow make_shared to access private constructor. public: ConstantExpression(double, bool, const type::SQLType*); ConstantExpression(const ConstantExpression&, const PrivateKey&); ConstantExpression& operator=(const ConstantExpression&) = delete; ~ConstantExpression() override; void prepare(SQLSelect&) override { NOTIMP; } void cleanup(SQLSelect&) override { NOTIMP; } // -- For WHERE using SQLExpression::eval; double eval(bool& missing) const override { missing = missing_; return value_; } bool isConstant() const override { return true; } bool isNumber() const override { NOTIMP; } // virtual SQLExpression* simplify(bool&); // virtual void title(const std::string&); // std::string title() const override; const type::SQLType* type() const override; // ---- std::shared_ptr clone() const override { return std::make_shared(*this, PrivateKey()); } std::shared_ptr reshift(int minColumnShift) const override { return clone(); } bool isAggregate() const override { return false; } // For select expression void output(SQLOutput& s) const override; void partialResult() override { NOTIMP; } virtual void expandStars(const std::vector>&, expression::Expressions&) override { NOTIMP; } bool isBitfield() const override { return isBitfield_; } BitfieldDef bitfieldDef() const { return bitfieldDef_; } bool hasMissingValue() const override { return hasMissingValue_; } double missingValue() const { return missingValue_; } protected: void print(std::ostream&) const override { NOTIMP; } bool isBitfield_; BitfieldDef bitfieldDef_; bool hasMissingValue_; double missingValue_; double value_; bool missing_; const type::SQLType& type_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace expression } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/expression/OrderByExpressions.cc0000664000175000017500000001017215161702250024007 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/OrderByExpressions.h" #include "eckit/utils/StringTools.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- const static std::vector nullAscending; OrderByExpressions::OrderByExpressions(const OrderByExpressions& o) : Expressions(o), ascending_(o.ascending_) {} OrderByExpressions::OrderByExpressions(const std::vector& ascending) : ascending_(ascending) {} OrderByExpressions::OrderByExpressions() : ascending_(nullAscending) {} OrderByExpressions::~OrderByExpressions() {} bool OrderByExpressions::operator<(const OrderByExpressions& o) const { size_t n = size(); // ASSERT(n == o.size()); ASSERT(ascending_.size() == n || ascending_.empty()); for (size_t i = 0; i < n; ++i) { bool asc = ascending_.empty() ? true : ascending_[i]; const SQLExpression& left = asc ? *(*this)[i] : *o[i]; const SQLExpression& right = asc ? *o[i] : *(*this)[i]; if (left.type()->getKind() == type::SQLType::stringType) { if (right.type()->getKind() != type::SQLType::stringType) { return false; } bool missing1 = false; bool missing2 = false; std::string v1(left.evalAsString(missing1)); std::string v2(right.evalAsString(missing2)); if (missing1 != missing2) { return missing1; } v1 = StringTools::trim(v1, "\t\n\v\f\r "); v2 = StringTools::trim(v2, "\t\n\v\f\r "); if (v1 != v2) { return (v1 < v2); } } else { bool leftMissing = false; bool rightMissing = false; double leftValue = left.eval(leftMissing); double rightValue = right.eval(rightMissing); if (leftMissing != rightMissing) { return leftMissing; } if (leftValue != rightValue) { return (leftValue < rightValue); } } } // They are equal return false; } bool OrderByExpressions::operator==(const OrderByExpressions& o) const { size_t n = size(); // ASSERT(n == o.size()); ASSERT(ascending_.size() == n || ascending_.empty()); for (size_t i = 0; i < n; ++i) { bool asc = ascending_[i]; const SQLExpression& left = asc ? *(*this)[i] : *o[i]; const SQLExpression& right = asc ? *o[i] : *(*this)[i]; if (left.type()->getKind() == type::SQLType::stringType) { if (right.type()->getKind() != type::SQLType::stringType) { return false; } bool missing1 = false; bool missing2 = false; std::string v1(left.evalAsString(missing1)); std::string v2(right.evalAsString(missing2)); if (missing1 != missing2) { return false; } v1 = StringTools::trim(v1, "\t\n\v\f\r "); v2 = StringTools::trim(v2, "\t\n\v\f\r "); if (v1 != v2) { return false; } } else { bool leftMissing = false; bool rightMissing = false; double leftValue = left.eval(leftMissing); double rightValue = right.eval(rightMissing); if (leftMissing != rightMissing) { return false; } if (leftValue != rightValue) { return false; } } } // They are equal return true; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/OrderByExpressions.h0000664000175000017500000000264015161702250023652 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// @date Nov 2011 /// @date Aug 2018 #ifndef eckit_sql_expression_OrderByExpressions_H #define eckit_sql_expression_OrderByExpressions_H #include "eckit/sql/expression/SQLExpressions.h" namespace eckit::sql::expression { /// @note This is fundamentally used only for the purpose of the SQLOrderOutput class //---------------------------------------------------------------------------------------------------------------------- class OrderByExpressions : public Expressions { public: OrderByExpressions(const OrderByExpressions& o); OrderByExpressions(const std::vector& ascending); OrderByExpressions(); ~OrderByExpressions(); bool operator<(const OrderByExpressions&) const; bool operator==(const OrderByExpressions&) const; private: const std::vector& ascending_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression #endif eckit-2.0.7/src/eckit/sql/expression/SQLExpressions.cc0000664000175000017500000000271515161702250023104 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/SQLExpressions.h" #include "eckit/sql/type/SQLType.h" namespace eckit::sql::expression { const type::SQLType* Expressions::type() const { return &type::SQLType::lookup("real"); } std::shared_ptr Expressions::clone() const { std::shared_ptr r = std::make_shared(this->size()); for (size_t i = 0; i < this->size(); ++i) { (*r)[i] = (*this)[i]->clone(); } return r; } std::shared_ptr Expressions::reshift(int minColumnShift_) const { // This is almost certainly not what you are trying to do here. // See reshift_expressions NOTIMP; } Expressions Expressions::reshift_expressions(int minColumnShift) const { Expressions shifted; for (auto& e : *this) { shifted.emplace_back(e->reshift(minColumnShift)); } return shifted; } void Expressions::print(std::ostream& o) const { o << "["; for (size_t i = 0; i < size(); ++i) { at(i)->print(o); o << ","; } o << "]"; } } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/SQLExpressionEvaluated.h0000664000175000017500000000413115161702250024410 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// @date Nov 2011 /// @date Aug 2018 #ifndef eckit_sql_expressiono_SQLExpressionEvaluated_H #define eckit_sql_expressiono_SQLExpressionEvaluated_H #include "eckit/sql/expression/SQLExpression.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- class SQLExpressionEvaluated : public SQLExpression { public: SQLExpressionEvaluated(SQLExpression&); ~SQLExpressionEvaluated() override; // Overriden void prepare(SQLSelect&) override; void updateType(SQLSelect&) override; void cleanup(SQLSelect&) override; double eval(bool& missing) const override; void eval(double* out, bool& missing) const override; std::string evalAsString(bool& missing) const override; bool isConstant() const override; bool isNumber() const override; std::shared_ptr simplify(bool&) override; std::shared_ptr clone() const override; std::shared_ptr reshift(int minColumnShift) const override; bool isAggregate() const override; const type::SQLType* type() const override; void output(SQLOutput& o) const override; protected: void print(std::ostream&) const override; friend std::ostream& operator<<(std::ostream& s, const SQLExpressionEvaluated& p) { p.print(s); return s; } const type::SQLType* type_; bool missing_; std::vector value_; double missingValue_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression #endif eckit-2.0.7/src/eckit/sql/expression/BitColumnExpression.h0000664000175000017500000000447415161702250024024 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 2003 #ifndef eckit_sql_BitColumnExpression_H #define eckit_sql_BitColumnExpression_H #include "eckit/sql/expression/ColumnExpression.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- class BitColumnExpression : public ColumnExpression { public: BitColumnExpression(const std::string&, const std::string&, SQLTable*); BitColumnExpression(const std::string&, const std::string&, const std::string&); BitColumnExpression(const BitColumnExpression&); ~BitColumnExpression(); std::shared_ptr clone() const override; std::shared_ptr reshift(int minColumnShift) const override; private: // No copy allowed BitColumnExpression& operator=(const BitColumnExpression&); protected: unsigned long mask_; unsigned long bitShift_; std::string field_; std::string name_; // -- Overridden methods void prepare(SQLSelect& sql) override; void updateType(SQLSelect& sql) override; // Use SQLExpression's eval rather than ColumnExpression's void eval(double* out, bool& missing) const override { SQLExpression::eval(out, missing); } double eval(bool& missing) const override; virtual void expandStars(const std::vector>&, expression::Expressions&) override; const eckit::sql::type::SQLType* type() const override; std::string tableColumnToFullname(const SQLColumn& column) const override; // friend std::ostream& operator<<(std::ostream& s,const BitColumnExpression& p) // { p.print(s); return s; } friend class SQLSelectFactory; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression #endif eckit-2.0.7/src/eckit/sql/expression/SQLExpression.h0000664000175000017500000000703415161702250022562 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLExpression.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLExpression_H #define SQLExpression_H #include #include #include "eckit/sql/SQLTypedefs.h" #include "eckit/sql/type/SQLType.h" namespace eckit::sql { // Forward declarations class SQLSelect; class SQLTable; class SQLOutput; namespace expression { class Expressions; class SQLExpression; class Dictionary; class SQLExpression : public std::enable_shared_from_this { public: SQLExpression(); virtual ~SQLExpression(); /// pre-prepare ideally wouldn't exist. To enable a two-pass prepare of SQL columns /// for SQLSelect, this is here. virtual void preprepare(SQLSelect&) {} virtual void prepare(SQLSelect&) = 0; virtual void updateType(SQLSelect&) {} virtual void cleanup(SQLSelect&) = 0; // @note Original SQL implementation used double for everything, but strings may // have more than 8 characters, so we need a way of getting bigger data out. // --> where relevant the eval(double*, bool&) method is used. Otherwise it just // forwards to the normal eval. virtual double eval(bool& missing) const = 0; virtual void eval(double* out, bool& missing) const; virtual std::string evalAsString(bool& missing) const; virtual bool andSplit(expression::Expressions&) { return false; } virtual void tables(std::set&) {} virtual bool isConstant() const = 0; virtual bool isNumber() const { return false; } virtual std::shared_ptr simplify(bool&); virtual void title(const std::string&); virtual std::string title() const; virtual const type::SQLType* type() const = 0; // non-owning, do not delete result // ---- virtual std::shared_ptr clone() const = 0; virtual std::shared_ptr reshift(int minColumnShift) const = 0; virtual bool isAggregate() const { return false; } // For select expression virtual void output(SQLOutput&) const; virtual void partialResult() {} virtual void expandStars(const std::vector>&, expression::Expressions&); virtual bool isBitfield() const { return isBitfield_; } BitfieldDef bitfieldDef() const { return bitfieldDef_; } virtual bool hasMissingValue() const { return hasMissingValue_; } double missingValue() const { return missingValue_; } static std::shared_ptr number(double); virtual void print(std::ostream&) const = 0; protected: SQLExpression(const SQLExpression&) = default; SQLExpression& operator=(const SQLExpression&) = default; bool isBitfield_; BitfieldDef bitfieldDef_; bool hasMissingValue_; double missingValue_; // bool isVector_; // Vector* vector_; private: std::string title_; friend std::ostream& operator<<(std::ostream& s, const SQLExpression& p) { p.print(s); return s; } }; } // namespace expression } // namespace eckit::sql // #include "eckit/sql/expression/SQLExpressions.h" using namespace eckit::sql::expression; #endif eckit-2.0.7/src/eckit/sql/expression/SQLExpressions.h0000664000175000017500000000600315161702250022740 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// @date Nov 11 #ifndef eckit_api_Expressions_H #define eckit_api_Expressions_H #include #include #include "eckit/exception/Exceptions.h" #include "eckit/sql/expression/SQLExpression.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- // n.b. shared pointer not unique_ptr. // needs to be copyable to be used by YACC using ExpressionsVector = std::vector>; class Expressions : public SQLExpression, public ExpressionsVector { public: Expressions() : ExpressionsVector() {} Expressions(size_t i) : ExpressionsVector(i, nullptr) {} Expressions(const Expressions&) = default; Expressions& operator=(const Expressions&) = default; Expressions(Expressions&&) = default; Expressions& operator=(Expressions&&) = default; void print(std::ostream& s) const override; friend std::ostream& operator<<(std::ostream& o, const Expressions& e) { e.print(o); return o; } ////////////////////////////////////////////////////////////////////////////////////// void prepare(SQLSelect&) override {} void cleanup(SQLSelect&) override {} const type::SQLType* type() const override; // -- For WHERE using SQLExpression::eval; double eval(bool& missing) const override { NOTIMP; } bool isConstant() const override { NOTIMP; } bool isNumber() const override { return false; } virtual bool isVector() const { return true; } virtual Expressions& vector() { return *this; } std::shared_ptr simplify(bool&) override { return shared_from_this(); } std::shared_ptr clone() const override; std::shared_ptr reshift(int minColumnShift_) const override; virtual Expressions reshift_expressions(int minColumnShift_) const; bool isAggregate() const override { return false; } // For select expression void output(SQLOutput&) const override { return NOTIMP; } void partialResult() override {} virtual void expandStars(const std::vector>&, expression::Expressions&) override { NOTIMP; } ////////////////////////////////////////////////////////////////////////////////////// }; using VectorOfExpressions = std::vector; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression #endif eckit-2.0.7/src/eckit/sql/expression/ParameterExpression.h0000664000175000017500000000327315161702250024044 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 2003 #ifndef eckit_sql_ParameterExpression_H #define eckit_sql_ParameterExpression_H #include "eckit/sql/expression/SQLExpression.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- class ParameterExpression : public SQLExpression { public: ParameterExpression(int); ParameterExpression(const ParameterExpression&); ~ParameterExpression(); std::shared_ptr clone() const override; std::shared_ptr reshift(int minColumnShift) const override { return clone(); } private: // No copy allowed ParameterExpression& operator=(const ParameterExpression&); // -- Members // None double value_; int which_; void print(std::ostream& s) const override; void prepare(SQLSelect& sql) override; void cleanup(SQLSelect& sql) override; using SQLExpression::eval; double eval(bool& missing) const override; const type::SQLType* type() const override; bool isConstant() const override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression #endif eckit-2.0.7/src/eckit/sql/expression/StringExpression.h0000664000175000017500000000406715161702250023374 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File StringExpression.h // Baudouin Raoult - ECMWF Dec 03 #ifndef eckit_sql_StringExpression_H #define eckit_sql_StringExpression_H #include "eckit/sql/expression/SQLExpression.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- class StringExpression : public SQLExpression { public: StringExpression(const std::string&); StringExpression(const StringExpression&); ~StringExpression(); std::shared_ptr clone() const override; std::shared_ptr reshift(int minColumnShift) const override; private: // No copy allowed StringExpression& operator=(const StringExpression&); std::string name_; std::vector value_; const type::SQLType* type_; // non-owning // -- Overridden methods void print(std::ostream& s) const override; virtual void expandStars(const std::vector>&, expression::Expressions&) override; void prepare(SQLSelect& sql) override; void cleanup(SQLSelect& sql) override; const type::SQLType* type() const override; double eval(bool& missing) const override; void eval(double* out, bool& missing) const override; std::string evalAsString(bool& missing) const override; bool isConstant() const override { return true; } bool isNumber() const override { return true; } void output(SQLOutput& o) const override; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression #endif eckit-2.0.7/src/eckit/sql/expression/NumberExpression.h0000664000175000017500000000335615161702250023356 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // @author Baudouin Raoult // @author Simon Smart // @date Dec 03 #ifndef eckit_sql_NumberExpression_H #define eckit_sql_NumberExpression_H #include "eckit/sql/expression/SQLExpression.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- class NumberExpression : public SQLExpression { public: NumberExpression(double value); NumberExpression(const NumberExpression&); ~NumberExpression(); std::shared_ptr clone() const override; std::shared_ptr reshift(int minColumnShift) const override; void value(double v) { value_ = v; } private: // No copy allowed NumberExpression& operator=(const NumberExpression&); double value_; // -- Overridden methods void print(std::ostream& s) const override; void prepare(SQLSelect& sql) override; void cleanup(SQLSelect& sql) override; const type::SQLType* type() const override; using SQLExpression::eval; double eval(bool& missing) const override; bool isConstant() const override { return true; } bool isNumber() const override { return true; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression #endif eckit-2.0.7/src/eckit/sql/expression/SQLExpression.cc0000664000175000017500000000516515161702250022723 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/SQLExpression.h" #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLOutput.h" #include "eckit/sql/expression/NumberExpression.h" #include "eckit/sql/expression/SQLExpressions.h" using namespace eckit; namespace eckit::sql::expression { double const MISSING_VALUE_REAL = -2147483647.0; // long const MISSING_VALUE_INT = 2147483647; SQLExpression::SQLExpression() : isBitfield_(false), hasMissingValue_(false), missingValue_(MISSING_VALUE_REAL) {} SQLExpression::~SQLExpression() {} void SQLExpression::eval(double* out, bool& missing) const { // In the common case we are just dealing with a double, so we can use the normal // eval function. // --> In the cases of strings, there may be more data. *out = eval(missing); } std::shared_ptr SQLExpression::number(double value) { return std::make_shared(value); } std::shared_ptr SQLExpression::simplify(bool& changed) { if (isConstant() && !isNumber()) { changed = true; bool missing = false; LOG_DEBUG_LIB(LibEcKit) << "SIMPLIFY " << *this << " to " << eval(missing) << std::endl; return std::make_shared(eval(missing)); } return nullptr; } std::string SQLExpression::evalAsString(bool& missing) const { bool m; double d = eval(m); if (m) { missing = true; } else { return this->type()->asString(&d); } return std::string(); } void SQLExpression::output(SQLOutput& s) const { bool missing = false; double d = eval(missing); s.outputReal(d, missing); } void SQLExpression::title(const std::string& t) { title_ = t; } std::string SQLExpression::title() const { if (title_.size()) { return title_; } std::ostringstream s; s << *this; return s.str(); } // const type::SQLType* SQLExpression::type() const { const type::SQLType& x = type::SQLType::lookup("real"); return &x; // } void SQLExpression::expandStars(const std::vector>&, expression::Expressions& e) { e.push_back(shared_from_this()); } } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/ParameterExpression.cc0000664000175000017500000000403715161702250024201 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/ParameterExpression.h" #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLSession.h" namespace eckit::sql::expression { //---------------------------------------------------------------------------------------------------------------------- ParameterExpression::ParameterExpression(int which) : SQLExpression(), value_(0), which_(which) { // don't use any Log::* here // std::cout << "new ParameterExpression " << name << std::endl; } ParameterExpression::ParameterExpression(const ParameterExpression& other) : SQLExpression(), value_(other.value_), which_(other.which_) {} std::shared_ptr ParameterExpression::ParameterExpression::clone() const { return std::make_shared(*this); } ParameterExpression::~ParameterExpression() {} // TODO: are only real parameters allowed? const type::SQLType* ParameterExpression::type() const { return &type::SQLType::lookup("real"); } double ParameterExpression::eval(bool& missing) const { return value_; } void ParameterExpression::prepare(SQLSelect& sql) { NOTIMP; // TODO: // value_ = SQLSession::current().getParameter(which_); // std::cout << "ParameterExpression " << name_ << " " << value_ << std::endl; } void ParameterExpression::cleanup(SQLSelect& sql) { value_ = 0; } void ParameterExpression::print(std::ostream& s) const { s << '?' << which_ << '=' << value_; } bool ParameterExpression::isConstant() const { return false; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression eckit-2.0.7/src/eckit/sql/expression/function/0000775000175000017500000000000015161702250021513 5ustar alastairalastaireckit-2.0.7/src/eckit/sql/expression/function/FunctionIN.cc0000664000175000017500000000300115161702250024030 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionIN.h" #include "eckit/sql/expression/function/FunctionEQ.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder inFunctionBuilder("in"); FunctionIN::FunctionIN(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), size_(args.size() - 1) {} FunctionIN::FunctionIN(const FunctionIN& other) : FunctionExpression(other.name_, other.args_), size_(other.args_.size() - 1) {} FunctionIN::~FunctionIN() {} const type::SQLType* FunctionIN::type() const { return &type::SQLType::lookup("real"); } // TODO: bool? std::shared_ptr FunctionIN::clone() const { return std::make_shared(*this); } double FunctionIN::eval(bool& missing) const { const SQLExpression& x = *args_[size_]; for (size_t i = 0; i < size_; ++i) { if (FunctionEQ::equal(x, *args_[i], missing)) { return true; } } return false; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionTIMESTAMP.h0000664000175000017500000000260515161702250024740 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionTIMESTAMP.h // ECMWF July 2010 #ifndef FunctionTIMESTAMP_H #define FunctionTIMESTAMP_H #include "eckit/sql/expression/function/FunctionIntegerExpression.h" namespace eckit::sql::expression::function { class FunctionTIMESTAMP : public FunctionIntegerExpression { public: FunctionTIMESTAMP(const std::string&, const expression::Expressions&); FunctionTIMESTAMP(const FunctionTIMESTAMP&); ~FunctionTIMESTAMP(); std::shared_ptr clone() const override; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; static int arity() { return 2; } private: FunctionTIMESTAMP& operator=(const FunctionTIMESTAMP&); // -- Overridden methods using FunctionIntegerExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionTIMESTAMP& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionFactory.cc0000664000175000017500000000774315161702250025152 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { //---------------------------------------------------------------------------------------------------------------------- FunctionFactory::FunctionFactory() {} FunctionFactory::~FunctionFactory() {} FunctionFactory& FunctionFactory::instance() { static FunctionFactory theInstance; return theInstance; } void FunctionFactory::enregister(const std::string& name, int arity, const FunctionBuilderBase* builder) { std::lock_guard lock(m_); auto key = std::make_pair(name, arity); auto it = builders_.find(key); ASSERT(it == builders_.end()); builders_[key] = builder; } void FunctionFactory::deregister(const std::string& name, int arity, const FunctionBuilderBase* builder) { std::lock_guard lock(m_); auto it = builders_.find(std::make_pair(name, arity)); ASSERT(it != builders_.end()); ASSERT(it->second == builder); builders_.erase(it); } std::vector FunctionFactory::functionsInfo() { std::lock_guard lock(m_); std::vector info; info.reserve(builders_.size()); for (const auto& b : builders_) { ASSERT(b.first.second == b.second->arity()); info.emplace_back(FuncInfo{b.first.first, b.second->arity(), b.second->help()}); } return info; } std::shared_ptr FunctionFactory::build(const std::string& name, const Expressions& args) const { std::lock_guard lock(m_); // Do lookup for builder including the arity (-1 for "any") auto it = builders_.find(std::make_pair(name, int(args.size()))); if (it == builders_.end()) { it = builders_.find(std::make_pair(name, -1)); } if (it == builders_.end()) { throw UserError(name + ": function not defined", Here()); } return it->second->make(name, args); } std::shared_ptr FunctionFactory::build(const std::string& name, std::shared_ptr arg1) { expression::Expressions args; args.push_back(arg1); return build(name, args); } std::shared_ptr FunctionFactory::build(const std::string& name, std::shared_ptr arg1, std::shared_ptr arg2) { expression::Expressions args; args.push_back(arg1); args.push_back(arg2); return build(name, args); } std::shared_ptr FunctionFactory::build(const std::string& name, std::shared_ptr arg1, std::shared_ptr arg2, std::shared_ptr arg3) { expression::Expressions args; args.push_back(arg1); args.push_back(arg2); args.push_back(arg3); return build(name, args); } FunctionBuilderBase::FunctionBuilderBase(const std::string& name, int arity, const char* help) : name_(name), arity_(arity), help_(help) { FunctionFactory::instance().enregister(name_, arity_, this); } FunctionBuilderBase::~FunctionBuilderBase() { FunctionFactory::instance().deregister(name_, arity_, this); } int FunctionBuilderBase::arity() const { return arity_; } std::string FunctionBuilderBase::help() const { return help_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionTDIFF.h0000664000175000017500000000265515161702250024236 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionTDIFF.h // ECMWF July 2010 #ifndef FunctionTDIFF_H #define FunctionTDIFF_H #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/expression/function/FunctionIntegerExpression.h" namespace eckit::sql::expression::function { /* Static self-registration */ class FunctionTDIFF : public FunctionIntegerExpression { public: FunctionTDIFF(const std::string&, const expression::Expressions&); FunctionTDIFF(const FunctionTDIFF&); ~FunctionTDIFF(); std::shared_ptr clone() const override; static int arity() { return 4; } private: // No copy allowed FunctionTDIFF& operator=(const FunctionTDIFF&); // -- Overridden methods const eckit::sql::type::SQLType* type() const override; using FunctionIntegerExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionTDIFF& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionNULL.cc0000664000175000017500000000257515161702250024313 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionNULL.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder nullFunctionBuilder("null"); static FunctionBuilder isnullFunctionBuilder("isnull"); FunctionNULL::FunctionNULL(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args) {} FunctionNULL::FunctionNULL(const FunctionNULL& other) : FunctionExpression(other.name_, other.args_) {} FunctionNULL::~FunctionNULL() {} std::shared_ptr FunctionNULL::clone() const { return std::make_shared(*this); } const type::SQLType* FunctionNULL::type() const { return &type::SQLType::lookup("real"); } // Don't set the missing flag double FunctionNULL::eval(bool&) const { bool missing = false; args_[0]->eval(missing); return missing; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionNE.h0000664000175000017500000000262415161702250023700 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionNE.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionNE_H #define FunctionNE_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionNE : public FunctionExpression { public: FunctionNE(const std::string&, const expression::Expressions&); FunctionNE(const FunctionNE&); ~FunctionNE(); static bool equal(const SQLExpression& l, const SQLExpression& r, bool& missing); std::shared_ptr clone() const override; static int arity() { return 2; } private: // No copy allowed FunctionNE& operator=(const FunctionNE&); double tmp_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionNE& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionIntegerExpression.h0000664000175000017500000000267615161702250027062 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionIntegerExpression.h // ECMWF July 2010 #ifndef FUNCTION_INTEGER_EXPRESSION_H #define FUNCTION_INTEGER_EXPRESSION_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql { class SQLOutput; namespace expression::function { class FunctionIntegerExpression : public FunctionExpression { public: static void registerIntegerFunctions(); FunctionIntegerExpression(const std::string&, const expression::Expressions&); ~FunctionIntegerExpression(); // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void output(SQLOutput& s) const override; static int arity() { return 1; } private: // No copy allowed FunctionIntegerExpression(const FunctionIntegerExpression&); FunctionIntegerExpression& operator=(const FunctionIntegerExpression&); // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionIntegerExpression& p) // { p.print(s); return s; } }; } // namespace expression::function } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionCOUNT.cc0000664000175000017500000000343115161702250024421 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionCOUNT.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder countFunctionBuilder("count"); const type::SQLType* FunctionCOUNT::type() const { const type::SQLType& x = type::SQLType::lookup("double"); return &x; } FunctionCOUNT::FunctionCOUNT(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), count_(0) {} FunctionCOUNT::FunctionCOUNT(const FunctionCOUNT& other) : FunctionExpression(other.name_, other.args_), count_(other.count_) {} std::shared_ptr FunctionCOUNT::clone() const { return std::make_shared(*this); } FunctionCOUNT::~FunctionCOUNT() {} double FunctionCOUNT::eval(bool& missing) const { // cout << "FunctionCOUNT " << count_ << std::endl; return count_; } void FunctionCOUNT::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); count_ = 0; } void FunctionCOUNT::cleanup(SQLSelect& sql) { FunctionExpression::cleanup(sql); count_ = 0; } void FunctionCOUNT::partialResult() { bool missing = false; args_[0]->eval(missing); if (!missing) { count_++; } // cout << "FunctionCOUNT::partialResult " << count_ << std::endl; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionRMS.cc0000664000175000017500000000367115161702250024200 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/expression/function/FunctionRMS.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder rmsFunctionBuilder("rms"); FunctionRMS::FunctionRMS(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), count_(0), squares_(0) {} FunctionRMS::FunctionRMS(const FunctionRMS& other) : FunctionExpression(other.name_, other.args_), count_(other.count_), squares_(other.squares_) {} FunctionRMS::~FunctionRMS() {} std::shared_ptr FunctionRMS::clone() const { return std::make_shared(*this); } const type::SQLType* FunctionRMS::type() const { return &type::SQLType::lookup("double"); } double FunctionRMS::eval(bool& missing) const { if (!count_) { missing = true; return 0; } return sqrt(squares_ / count_); } void FunctionRMS::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); count_ = 0; squares_ = 0; } void FunctionRMS::cleanup(SQLSelect& sql) { // cout << "Cleanup FunctionRMS " << count_ << " " << value_ << std::endl; FunctionExpression::cleanup(sql); squares_ = 0; count_ = 0; } void FunctionRMS::partialResult() { bool missing = false; double value = args_[0]->eval(missing); if (!missing) { squares_ += value * value; count_++; } // else cout << "missing" << std::endl; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionFIRST.cc0000664000175000017500000000370715161702250024426 0ustar alastairalastair/* * (C) Copyright 1996-2016 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/sql/expression/function/FunctionFIRST.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder firstFunctionBuilder("first"); FunctionFIRST::FunctionFIRST(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), value_(DBL_MAX), notFirst_(false) {} FunctionFIRST::FunctionFIRST(const FunctionFIRST& other) : FunctionExpression(other.name_, other.args_), value_(other.value_), notFirst_(other.notFirst_) {} std::shared_ptr FunctionFIRST::clone() const { return std::make_shared(*this); } const eckit::sql::type::SQLType* FunctionFIRST::type() const { return args_[0]->type(); } FunctionFIRST::~FunctionFIRST() {} double FunctionFIRST::eval(bool& missing) const { if (value_ == DBL_MAX) { missing = true; } return value_; } void FunctionFIRST::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); value_ = DBL_MAX; } void FunctionFIRST::cleanup(SQLSelect& sql) { FunctionExpression::cleanup(sql); value_ = DBL_MAX; } void FunctionFIRST::output(SQLOutput& s) const { bool missing(false); double d(eval(missing)); type()->output(s, d, missing); } void FunctionFIRST::partialResult() { if (notFirst_) { return; } bool missing(false); value_ = (args_[0]->eval(missing)); notFirst_ = true; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionNVL.h0000664000175000017500000000246615161702250024041 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionNVL.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionNVL_H #define FunctionNVL_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionNVL : public FunctionExpression { public: FunctionNVL(const std::string&, const expression::Expressions&); FunctionNVL(const FunctionNVL&); ~FunctionNVL(); std::shared_ptr clone() const override; static int arity() { return 2; } private: // No copy allowed FunctionNVL& operator=(const FunctionNVL&); // -- Overridden methods const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionNVL& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionJOIN.h0000664000175000017500000000250115161702250024127 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionJOIN.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionJOIN_H #define FunctionJOIN_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionJOIN : public FunctionExpression { public: FunctionJOIN(const std::string&, const expression::Expressions&); FunctionJOIN(const FunctionJOIN&); ~FunctionJOIN(); std::shared_ptr clone() const override; static int arity() { return 2; } private: // No copy allowed FunctionJOIN& operator=(const FunctionJOIN&); // -- Overridden methods const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionJOIN& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionOR.cc0000664000175000017500000000344115161702250024052 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionOR.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder orFunctionBuilder("or"); FunctionOR::FunctionOR(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args) {} FunctionOR::FunctionOR(const FunctionOR& other) : FunctionExpression(other.name_, other.args_) {} FunctionOR::~FunctionOR() {} std::shared_ptr FunctionOR::clone() const { return std::make_shared(*this); } const type::SQLType* FunctionOR::type() const { return &type::SQLType::lookup("real"); } double FunctionOR::eval(bool& missing) const { return args_[0]->eval(missing) || args_[1]->eval(missing); } std::shared_ptr FunctionOR::simplify(bool& changed) { std::shared_ptr x = FunctionExpression::simplify(changed); if (x) { return x; } for (int i = 0; i < 2; i++) { bool missing = false; if (args_[i]->isConstant()) { if (args_[i]->eval(missing)) { std::cout << "SYMPLIFY " << *this << " to 1" << std::endl; ; changed = true; return SQLExpression::number(1); } } } return nullptr; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionMAX.h0000664000175000017500000000303715161702250024022 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionMAX.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionMAX_H #define FunctionMAX_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionMAX : public FunctionExpression { public: FunctionMAX(const std::string&, const expression::Expressions&); FunctionMAX(const FunctionMAX&); ~FunctionMAX(); std::shared_ptr clone() const override; static int arity() { return 1; } private: // No copy allowed FunctionMAX& operator=(const FunctionMAX&); double value_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; using FunctionExpression::eval; double eval(bool& missing) const override; bool isAggregate() const override { return true; } void output(SQLOutput&) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionMAX& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionNE.cc0000664000175000017500000000361615161702250024040 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionNE.h" #include "eckit/sql/expression/ColumnExpression.h" #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/type/SQLType.h" #include "eckit/utils/StringTools.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder neFunctionBuilder("<>"); using namespace eckit::sql::type; const type::SQLType* FunctionNE::type() const { return &type::SQLType::lookup("double"); } FunctionNE::FunctionNE(const FunctionNE& other) : FunctionExpression(other.name_, other.args_), tmp_(other.tmp_) {} FunctionNE::FunctionNE(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), tmp_(0) {} std::shared_ptr FunctionNE::clone() const { return std::make_shared(*this); } FunctionNE::~FunctionNE() {} bool FunctionNE::equal(const SQLExpression& l, const SQLExpression& r, bool& missing) { if (l.type()->getKind() == SQLType::stringType) { std::string v1(l.evalAsString(missing)); std::string v2(r.evalAsString(missing)); if (missing) { return false; } v1 = StringTools::trim(v1, "\t\n\v\f\r "); v2 = StringTools::trim(v2, "\t\n\v\f\r "); return (v1 != v2); } return l.eval(missing) != r.eval(missing); } double FunctionNE::eval(bool& missing) const { return equal(*args_[0], *args_[1], missing); } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionTHIN.cc0000664000175000017500000000363115161702250024275 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file FunctionTHIN.h /// (C) ECMWF July 2010 #include "eckit/sql/expression/function/FunctionTHIN.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/SQLTable.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder thinFunctionBuilder("thin"); FunctionTHIN::FunctionTHIN(const std::string& name, const expression::Expressions& args) : FunctionIntegerExpression(name, args), count_{nullptr} {} FunctionTHIN::FunctionTHIN(const FunctionTHIN& other) : FunctionIntegerExpression(other.name_, other.args_), count_(other.count_) {} FunctionTHIN::~FunctionTHIN() {} std::shared_ptr FunctionTHIN::clone() const { return std::make_shared(*this); } const eckit::sql::type::SQLType* FunctionTHIN::type() const { return &eckit::sql::type::SQLType::lookup("integer"); } void FunctionTHIN::print(std::ostream& s) const { s << "THIN()"; } double FunctionTHIN::eval(bool& missing) const { int every_nth = (int)args_[0]->eval(missing); if ((*count_) % every_nth == 0) { return 1.0; } return 0.0; } void FunctionTHIN::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); count_ = &sql.count_; } void FunctionTHIN::cleanup(SQLSelect& sql) {} bool FunctionTHIN::isConstant() const { return false; } std::shared_ptr FunctionTHIN::simplify(bool&) { return nullptr; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionDOTP.cc0000664000175000017500000000345715161702250024307 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionDOTP.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder dotpFunctionBuilder("dotp"); const type::SQLType* FunctionDOTP::type() const { return &type::SQLType::lookup("double"); } FunctionDOTP::FunctionDOTP(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), value_(0), resultNULL_(true) {} FunctionDOTP::FunctionDOTP(const FunctionDOTP& other) : FunctionExpression(other.name_, other.args_), value_(other.value_), resultNULL_(other.resultNULL_) {} std::shared_ptr FunctionDOTP::clone() const { return std::make_shared(*this); } FunctionDOTP::~FunctionDOTP() {} double FunctionDOTP::eval(bool& missing) const { if (resultNULL_) { missing = true; } return value_; } void FunctionDOTP::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); value_ = 0; } void FunctionDOTP::cleanup(SQLSelect& sql) { FunctionExpression::cleanup(sql); value_ = 0; } void FunctionDOTP::partialResult() { bool missing = false; double x = args_[0]->eval(missing); double y = args_[1]->eval(missing); if (!missing) { value_ += x * y; resultNULL_ = false; } } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionJOIN.cc0000664000175000017500000000251015161702250024265 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionJOIN.h" #include "eckit/sql/expression/ColumnExpression.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder joinFunctionBuilder("join"); FunctionJOIN::FunctionJOIN(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args) {} FunctionJOIN::FunctionJOIN(const FunctionJOIN& other) : FunctionExpression(other.name_, other.args_) {} std::shared_ptr FunctionJOIN::clone() const { return std::make_shared(*this); } FunctionJOIN::~FunctionJOIN() {} const type::SQLType* FunctionJOIN::type() const { return &type::SQLType::lookup("real"); } double FunctionJOIN::eval(bool& missing) const { return args_[0]->eval(missing) == args_[1]->eval(missing); } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionAVG.h0000664000175000017500000000314115161702250024006 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionAVG.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionAVG_H #define FunctionAVG_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionAVG : public FunctionExpression { public: FunctionAVG(const std::string&, const expression::Expressions&); FunctionAVG(const FunctionAVG&); ~FunctionAVG(); std::shared_ptr clone() const override; static int arity() { return 1; } static const char* help() { return "Average (aggregate function)"; } private: // No copy allowed FunctionAVG& operator=(const FunctionAVG&); // -- Members unsigned long long count_; double value_; // -- Overridden methods const type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; using FunctionExpression::eval; double eval(bool& missing) const override; bool isAggregate() const override { return true; } // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionAVG& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionAVG.cc0000664000175000017500000000371215161702250024150 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionAVG.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder avgFunctionBuilder("avg"); static FunctionBuilder meanFunctionBuilder("mean"); FunctionAVG::FunctionAVG(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), count_(0), value_(0) {} FunctionAVG::FunctionAVG(const FunctionAVG& other) : FunctionExpression(other.name_, other.args_), count_(other.count_), value_(other.value_) {} std::shared_ptr FunctionAVG::clone() const { return std::make_shared(*this); } const type::SQLType* FunctionAVG::type() const { return &type::SQLType::lookup("double"); } FunctionAVG::~FunctionAVG() {} double FunctionAVG::eval(bool& missing) const { if (!count_) { missing = true; return 0; } return value_ / count_; } void FunctionAVG::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); value_ = 0; count_ = 0; } void FunctionAVG::cleanup(SQLSelect& sql) { // cout << "Cleanup FunctionAVG " << count_ << " " << value_ << std::endl; FunctionExpression::cleanup(sql); value_ = 0; count_ = 0; } void FunctionAVG::partialResult() { bool missing = false; double value = args_[0]->eval(missing); if (!missing) { value_ += value; count_++; } // else cout << "missing" << std::endl; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionTIMESTAMP.cc0000664000175000017500000000366515161702250025105 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/expression/function/FunctionTIMESTAMP.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder timestampFunctionBuilder("timestamp"); FunctionTIMESTAMP::FunctionTIMESTAMP(const std::string& name, const expression::Expressions& args) : FunctionIntegerExpression(name, args) {} FunctionTIMESTAMP::FunctionTIMESTAMP(const FunctionTIMESTAMP& other) : FunctionIntegerExpression(other.name_, other.args_) {} std::shared_ptr FunctionTIMESTAMP::clone() const { return std::make_shared(*this); } FunctionTIMESTAMP::~FunctionTIMESTAMP() {} double FunctionTIMESTAMP::eval(bool& missing) const { double indate = args_[0]->eval(missing); double intime = args_[1]->eval(missing); // Merge "YYYYMMDD" and "HHMMSS" into "YYYYMMDDHHMMSS" double outstamp = 0; if (indate >= 0 && indate <= 99991231 && intime >= 0 && intime <= 240000) { long long int lldate = (long long int)indate; long long int lltime = (long long int)intime; long long int tstamp = lldate * 1000000ll + lltime; outstamp = tstamp; outstamp = trunc(outstamp); } else { missing = true; } return outstamp; } const eckit::sql::type::SQLType* FunctionTIMESTAMP::type() const { return &eckit::sql::type::SQLType::lookup("integer"); } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionNOT_IN.cc0000664000175000017500000000311615161702250024557 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionNOT_IN.h" #include "eckit/sql/expression/function/FunctionEQ.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder not_inFunctionBuilder("not_in"); FunctionNOT_IN::FunctionNOT_IN(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), size_(args.size() - 1) {} FunctionNOT_IN::FunctionNOT_IN(const FunctionNOT_IN& other) : FunctionExpression(other.name_, other.args_), size_(other.args_.size() - 1) {} FunctionNOT_IN::~FunctionNOT_IN() {} std::shared_ptr FunctionNOT_IN::clone() const { return std::make_shared(*this); } const type::SQLType* FunctionNOT_IN::type() const { return &type::SQLType::lookup("double"); } double FunctionNOT_IN::eval(bool& missing) const { const SQLExpression& x = *args_[size_]; for (int i = 0; i < size_; ++i) { args_[i]->eval(missing); if (FunctionEQ::equal(x, *args_[i], missing)) { return false; } } return true; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionFIRST.h0000664000175000017500000000315515161702250024265 0ustar alastairalastair/* * (C) Copyright 1996-2016 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionFIRST.h // Piotr Kuchta - ECMWF Nov 2016 #ifndef eckit_sql_function_FunctionFIRST_H #define eckit_sql_function_FunctionFIRST_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionFIRST : public FunctionExpression { public: FunctionFIRST(const std::string&, const expression::Expressions&); FunctionFIRST(const FunctionFIRST&); ~FunctionFIRST(); std::shared_ptr clone() const override; static int arity() { return 1; } private: // No copy allowed FunctionFIRST& operator=(const FunctionFIRST&); double value_; bool notFirst_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; using FunctionExpression::eval; double eval(bool& missing) const override; bool isAggregate() const override { return true; } void output(SQLOutput&) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionFIRST& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionMIN.cc0000664000175000017500000000362715161702250024163 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/expression/function/FunctionMIN.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder minFunctionBuilder("min"); FunctionMIN::FunctionMIN(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), value_(DBL_MAX) {} FunctionMIN::FunctionMIN(const FunctionMIN& other) : FunctionExpression(other.name_, other.args_), value_(other.value_) {} std::shared_ptr FunctionMIN::clone() const { return std::make_shared(*this); } const eckit::sql::type::SQLType* FunctionMIN::type() const { return args_[0]->type(); } FunctionMIN::~FunctionMIN() {} double FunctionMIN::eval(bool& missing) const { if (value_ == DBL_MAX) { missing = true; } return value_; } void FunctionMIN::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); value_ = DBL_MAX; } void FunctionMIN::cleanup(SQLSelect& sql) { FunctionExpression::cleanup(sql); value_ = DBL_MAX; } void FunctionMIN::output(SQLOutput& s) const { bool missing = false; double d = eval(missing); type()->output(s, d, missing); } void FunctionMIN::partialResult() { bool missing = false; double value = args_[0]->eval(missing); if (!missing) { if (value < value_) { value_ = value; } } } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionNOT_NULL.cc0000664000175000017500000000302315161702250025020 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionNOT_NULL.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder not_nullFunctionBuilder("not_null"); FunctionNOT_NULL::FunctionNOT_NULL(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args) {} FunctionNOT_NULL::FunctionNOT_NULL(const FunctionNOT_NULL& other) : FunctionExpression(other.name_, other.args_) {} FunctionNOT_NULL::~FunctionNOT_NULL() {} std::shared_ptr FunctionNOT_NULL::clone() const { return std::make_shared(*this); } const type::SQLType* FunctionNOT_NULL::type() const { return &type::SQLType::lookup("real"); } // Don't set the missing flags double FunctionNOT_NULL::eval(bool&) const { bool missing = false; #if 0 std::cout << "FunctionNOT_NULL " << (*args_[0]) << " " << args_[0]->eval(missing); std::cout << " missing = " << missing << std::endl; #else args_[0]->eval(missing); #endif return !missing; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionSUM.cc0000664000175000017500000000336015161702250024176 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionSUM.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder sumFunctionBuilder("sum"); FunctionSUM::FunctionSUM(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), value_(0), resultNULL_(true) {} FunctionSUM::FunctionSUM(const FunctionSUM& other) : FunctionExpression(other.name_, other.args_), value_(other.value_), resultNULL_(other.resultNULL_) {} FunctionSUM::~FunctionSUM() {} std::shared_ptr FunctionSUM::clone() const { return std::make_shared(*this); } const type::SQLType* FunctionSUM::type() const { return &type::SQLType::lookup("double"); } double FunctionSUM::eval(bool& missing) const { if (resultNULL_) { missing = true; } return value_; } void FunctionSUM::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); value_ = 0; } void FunctionSUM::cleanup(SQLSelect& sql) { FunctionExpression::cleanup(sql); value_ = 0; } void FunctionSUM::partialResult() { bool missing = false; double value = args_[0]->eval(missing); if (!missing) { value_ += value; resultNULL_ = false; } } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionAND.h0000664000175000017500000000332415161702250023776 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // @author Baudouin Raoult // @author Simon Smart // @date December 2003 // @date January 2019 #ifndef FunctionAND_H #define FunctionAND_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { //---------------------------------------------------------------------------------------------------------------------- class FunctionAND : public FunctionExpression { public: FunctionAND(const std::string&, const expression::Expressions&); FunctionAND(const FunctionAND&); ~FunctionAND(); std::shared_ptr clone() const override; static int arity() { return 2; } static const char* help() { return ""; } private: // No copy allowed FunctionAND& operator=(const FunctionAND&); const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; std::shared_ptr simplify(bool&) override; bool andSplit(expression::Expressions&) override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionAND& p) // { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionEQ.cc0000664000175000017500000000560015161702250024036 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionEQ.h" #include "eckit/sql/expression/ColumnExpression.h" #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/type/SQLType.h" #include "eckit/utils/StringTools.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder eqFunctionBuilder("="); using namespace eckit::sql::type; const type::SQLType* FunctionEQ::type() const { return &type::SQLType::lookup("double"); } FunctionEQ::FunctionEQ(const FunctionEQ& other) : FunctionExpression(other.name_, other.args_), tmp_(other.tmp_) {} FunctionEQ::FunctionEQ(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), tmp_(0) {} std::shared_ptr FunctionEQ::clone() const { return std::make_shared(*this); } FunctionEQ::~FunctionEQ() {} void FunctionEQ::trimStringInDouble(char*& p, size_t& len) { // n.b. Duplicated into odb::WriterDispatchingIterator // TODO: Put somewhere better. len = 0; for (; len < sizeof(double) && isprint(p[len]); ++len) { ; } for (; len > 0 && isspace(p[len - 1]); --len) { ; } size_t plen = len; for (char* pp = p; isspace(*p) && p < pp + plen;) { ++p; --len; } } bool FunctionEQ::equal(const SQLExpression& l, const SQLExpression& r, bool& missing) { if (l.type()->getKind() == SQLType::stringType) { std::string v1(l.evalAsString(missing)); std::string v2(r.evalAsString(missing)); if (missing) { return false; } v1 = StringTools::trim(v1, "\t\n\v\f\r "); v2 = StringTools::trim(v2, "\t\n\v\f\r "); return (v1 == v2); } return l.eval(missing) == r.eval(missing); } double FunctionEQ::eval(bool& missing) const { return equal(*args_[0], *args_[1], missing); } std::shared_ptr FunctionEQ::simplify(bool& changed) { std::shared_ptr x = FunctionExpression::simplify(changed); if (x) { return x; } ColumnExpression* a = dynamic_cast(args_[0].get()); ColumnExpression* b = dynamic_cast(args_[1].get()); if (a && b) { return FunctionFactory::instance().build("join", args_[0], args_[1]); } // if (args_[0]->isConstant() && !args_[1]->isConstant()) { std::swap(args_[0], args_[1]); } return nullptr; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionMATCH.h0000664000175000017500000000316215161702250024230 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file FunctionMATCH.h /// Piotr Kuchta - ECMWF October 2015 #ifndef FunctionMATCH_H #define FunctionMATCH_H #include #include #include "eckit/sql/SQLAST.h" #include "eckit/sql/expression/function/FunctionExpression.h" #include "eckit/sql/type/SQLType.h" namespace eckit { namespace sql { namespace expression { namespace function { class FunctionMATCH : public FunctionExpression { public: FunctionMATCH(const std::string&, const eckit::sql::expression::Expressions&, const eckit::sql::SelectAST&); FunctionMATCH(const FunctionMATCH&); ~FunctionMATCH(); FunctionMATCH& operator=(const FunctionMATCH&); std::shared_ptr clone() const override; void collect(const std::vector&); private: size_t size_; const SelectAST subquery_; std::vector > subqueryResult_; const eckit::sql::type::SQLType* type() const override; double eval(bool& missing) const override; virtual void prepare(SQLSelect& sql); // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionMATCH& p) // { p.print(s); return s; } }; } // namespace function } // namespace expression } // namespace sql } // namespace eckit #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionDOTP.h0000664000175000017500000000300615161702250024137 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionDOTP.h // ECMWF July 2010 #ifndef FunctionDOTP_H #define FunctionDOTP_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionDOTP : public FunctionExpression { public: FunctionDOTP(const std::string&, const expression::Expressions&); FunctionDOTP(const FunctionDOTP&); ~FunctionDOTP(); std::shared_ptr clone() const override; static int arity() { return 2; } private: // No copy allowed FunctionDOTP& operator=(const FunctionDOTP&); double value_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; using FunctionExpression::eval; double eval(bool& missing) const override; bool isAggregate() const override { return true; } bool resultNULL_; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionDOTP& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionSUM.h0000664000175000017500000000304715161702250024042 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionSUM.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionSUM_H #define FunctionSUM_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionSUM : public FunctionExpression { public: FunctionSUM(const std::string&, const expression::Expressions&); FunctionSUM(const FunctionSUM&); ~FunctionSUM(); std::shared_ptr clone() const override; static int arity() { return 1; } private: // No copy allowed FunctionSUM& operator=(const FunctionSUM&); unsigned long long count_; double value_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; using FunctionExpression::eval; double eval(bool& missing) const override; bool isAggregate() const override { return true; } bool resultNULL_; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionSUM& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionJULIAN_SECONDS.cc0000664000175000017500000000372115161702250025673 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionJULIAN_SECONDS.h" #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/types/Date.h" #define trunc(x) ((x) - fmod((x), 1)) namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder julian_secondsFunctionBuilder("julian_seconds"); FunctionJULIAN_SECONDS::FunctionJULIAN_SECONDS(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args) {} FunctionJULIAN_SECONDS::FunctionJULIAN_SECONDS(const FunctionJULIAN_SECONDS& other) : FunctionExpression(other.name_, other.args_) {} std::shared_ptr FunctionJULIAN_SECONDS::clone() const { return std::make_shared(*this); } FunctionJULIAN_SECONDS::~FunctionJULIAN_SECONDS() {} const type::SQLType* FunctionJULIAN_SECONDS::type() const { return &type::SQLType::lookup("real"); } double FunctionJULIAN_SECONDS::eval(bool& missing) const { int indate = (int)args_[0]->eval(missing); int intime = (int)args_[1]->eval(missing); // TODO: shold we return MISSING_VALUE_INT in case missing == true here? int year = indate / 10000; int month = (indate % 10000) / 100; int day = indate % 100; int hour = intime / 10000; int min = (intime % 10000) / 100; int sec = intime % 100; // " Julianday * 24 * 60 * 60 + hh * 3600 + mm * 60 + ss "; return eckit::Date(year, month, day).julian() * 24 * 60 * 60 + hour * 3600 + min * 60 + sec; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionRLIKE.h0000664000175000017500000000314015161702250024236 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \File FunctionRLIKE.h /// Piotr Kuchta - ECMWF Sep 2014 #ifndef FunctionRLIKE_H #define FunctionRLIKE_H #include #include "eckit/sql/expression/function/FunctionExpression.h" #include "eckit/utils/Regex.h" namespace eckit::sql::expression::function { class FunctionRLIKE : public FunctionExpression { public: FunctionRLIKE(const std::string&, const expression::Expressions&); FunctionRLIKE(const FunctionRLIKE&); ~FunctionRLIKE(); bool match(const SQLExpression& l, const SQLExpression& r, bool& missing) const; static void trimStringInDouble(char*& p, size_t& len); std::shared_ptr clone() const override; void prepare(SQLSelect&) override; static int arity() { return 2; } private: // No copy allowed FunctionRLIKE& operator=(const FunctionRLIKE&); std::unique_ptr re_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionRLIKE& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionIntegerExpression.cc0000664000175000017500000001155615161702250027215 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/sql/SQLOutput.h" #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/expression/function/FunctionIntegerExpression.h" #include "eckit/utils/Translator.h" #define ftrunc(x) ((x) - fmod((x), 1)) #define F90nint(x) (((x) > 0) ? (int)((x) + 0.5) : (int)((x) - 0.5)) namespace eckit::sql::expression::function { FunctionIntegerExpression::FunctionIntegerExpression(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args) {} FunctionIntegerExpression::~FunctionIntegerExpression() {} const eckit::sql::type::SQLType* FunctionIntegerExpression::type() const { return &eckit::sql::type::SQLType::lookup("integer"); } void FunctionIntegerExpression::output(SQLOutput& s) const { bool missing = false; double v = eval(missing); s.outputInt(v, missing); // Log::info() << "FunctionIntegerExpression::output: v=" << v << ", missing=" << missing << std::endl; } //---------------------------------------------------------------------------------------------------------------------- // TODO: This is REALLY the wrong place for this to be. constexpr double DEFAULT_MDI = 2147483647; template class MathFunctionIntegerExpression_1 : public FunctionIntegerExpression { public: // methods MathFunctionIntegerExpression_1(const std::string& name, const expression::Expressions& args) : FunctionIntegerExpression(name, args) { this->missingValue_ = DEFAULT_MDI; } private: // methods using FunctionIntegerExpression::eval; double eval(bool& m) const { bool missing = false; double v = args_[0]->eval(missing); if (missing) { m = missing; return this->missingValue_; } return FN(v); } std::shared_ptr clone() const { return std::make_shared>(this->name_, this->args_); } }; //---------------------------------------------------------------------------------------------------------------------- inline double year(double x) { return (double)((int)((x) / 10000)); } inline double month(double x) { return (double)(((int)((x) / 100)) % 100); } inline double day(double x) { return (double)(((int)(x)) % 100); } inline double hour(double x) { return (double)((int)((x) / 10000)); } inline double minute(double x) { return (double)(((int)((x) / 100)) % 100); } inline double minutes(double x) { return minute(x); } inline double second(double x) { return (double)(((int)(x)) % 100); } inline double seconds(double x) { return second(x); } inline double Func_ftrunc(double x) { return (double)(ftrunc(x)); } inline double Func_dnint(double x) { return (double)(F90nint(x)); } inline double Func_dint(double x) { return (double)(ftrunc(x)); } inline double Func_ceil(double x) { return (double)(ceil(x)); } inline double Func_floor(double x) { return (double)(floor(x)); } inline double Func_atoi(double x) { return (double)atoi(Translator()(x).c_str()); } //---------------------------------------------------------------------------------------------------------------------- /* Static self-registration */ template using IntegerFunctionBuilder = FunctionBuilder>; static IntegerFunctionBuilder yearFunctionBuilder("year"); static IntegerFunctionBuilder monthFunctionBuilder("month"); static IntegerFunctionBuilder dayFunctionBuilder("day"); static IntegerFunctionBuilder hourFunctionBuilder("hour"); static IntegerFunctionBuilder minuteFunctionBuilder("minute"); static IntegerFunctionBuilder minutesFunctionBuilder("minutes"); static IntegerFunctionBuilder secondFunctionBuilder("second"); static IntegerFunctionBuilder secondsFunctionBuilder("seconds"); static IntegerFunctionBuilder ftruncFunctionBuilder("trunc"); static IntegerFunctionBuilder dnintFunctionBuilder("nint"); static IntegerFunctionBuilder dintFunctionBuilder("int"); static IntegerFunctionBuilder ceilFunctionBuilder("ceil"); static IntegerFunctionBuilder floorFunctionBuilder("floor"); static IntegerFunctionBuilder atoiFunctionBuilder("atoi"); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionExpression.cc0000664000175000017500000000753015161702250025674 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { //---------------------------------------------------------------------------------------------------------------------- FunctionExpression::FunctionExpression(const std::string& name, const expression::Expressions& args) : name_(name), args_(args) { // never use any logging here (Log::*) // std::cout << "new FunctionExpression " << name << std::endl; } FunctionExpression::FunctionExpression(const FunctionExpression& other) : name_(other.name_), args_(other.args_) {} const type::SQLType* FunctionExpression::type() const { return &type::SQLType::lookup("double"); } std::shared_ptr FunctionExpression::reshift(int minColumnShift) const { std::shared_ptr shifted = clone(); static_cast(*shifted).shiftArgs(minColumnShift); return shifted; } FunctionExpression::~FunctionExpression() {} void FunctionExpression::preprepare(SQLSelect& sql) { for (expression::Expressions::iterator j = args_.begin(); j != args_.end(); ++j) { (*j)->preprepare(sql); } } void FunctionExpression::prepare(SQLSelect& sql) { for (expression::Expressions::iterator j = args_.begin(); j != args_.end(); ++j) { (*j)->prepare(sql); } } void FunctionExpression::updateType(SQLSelect& sql) { for (expression::Expressions::iterator j = args_.begin(); j != args_.end(); ++j) { (*j)->updateType(sql); } } void FunctionExpression::cleanup(SQLSelect& sql) { for (expression::Expressions::iterator j = args_.begin(); j != args_.end(); ++j) { (*j)->cleanup(sql); } } void FunctionExpression::partialResult() { for (expression::Expressions::iterator j = args_.begin(); j != args_.end(); ++j) { (*j)->partialResult(); } } std::shared_ptr FunctionExpression::simplify(bool& changed) { for (std::shared_ptr& arg : args_) { std::shared_ptr simplifiedArg = arg->simplify(changed); if (simplifiedArg) { arg = simplifiedArg; std::cout << "SIMPLIFY " << *this << std::endl; changed = true; } } return SQLExpression::simplify(changed); } bool FunctionExpression::isConstant() const { for (expression::Expressions::const_iterator j = args_.begin(); j != args_.end(); ++j) { if (!(*j)->isConstant()) { return false; } } return true; } bool FunctionExpression::isAggregate() const { for (expression::Expressions::const_iterator j = args_.begin(); j != args_.end(); ++j) { if ((*j)->isAggregate()) { return true; } } return false; } void FunctionExpression::print(std::ostream& s) const { s << name_; s << '('; for (expression::Expressions::const_iterator j = args_.begin(); j != args_.end(); ++j) { if (j != args_.begin()) { s << ','; } s << *(*j); } s << ')'; } void FunctionExpression::tables(std::set& t) { for (expression::Expressions::iterator j = args_.begin(); j != args_.end(); ++j) { (*j)->tables(t); } } void FunctionExpression::shiftArgs(int minColumnShift) { args_ = args_.reshift_expressions(minColumnShift); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionVAR.h0000664000175000017500000000312115161702250024017 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionVAR.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionVAR_H #define FunctionVAR_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionVAR : public FunctionExpression { public: FunctionVAR(const std::string&, const expression::Expressions&); FunctionVAR(const FunctionVAR&); ~FunctionVAR(); // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; bool isAggregate() const override { return true; } std::shared_ptr clone() const override; static int arity() { return 1; } protected: // -- Overridden methods using FunctionExpression::eval; double eval(bool& missing) const override; private: // No copy allowed FunctionVAR& operator=(const FunctionVAR&); unsigned long long count_; double value_; double squares_; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionVAR& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionMATCH.cc0000664000175000017500000000466515161702250024377 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/types/Types.h" #include "eckit/sql/SQLMATCHSubquerySession.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/SQLStatement.h" #include "eckit/sql/expression/function/FunctionEQ.h" #include "eckit/sql/expression/function/FunctionMATCH.h" #include #include namespace eckit { namespace sql { namespace expression { namespace function { using namespace eckit; using namespace std; FunctionMATCH::FunctionMATCH(const std::string& name, const expression::Expressions& args, const SelectAST& selectAST) : FunctionExpression(name, args), size_(args.size()), subquery_(selectAST), subqueryResult_() {} FunctionMATCH::FunctionMATCH(const FunctionMATCH& other) : FunctionExpression(other.name_, other.args_), size_(other.args_.size()), subquery_(other.subquery_), subqueryResult_(other.subqueryResult_) {} FunctionMATCH::~FunctionMATCH() {} void FunctionMATCH::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); SQLMATCHSubquerySession session(*this); SQLSelect* select(session.selectFactory().create(session, subquery_)); session.execute(dynamic_cast(*select)); std::stable_sort(subqueryResult_.begin(), subqueryResult_.end()); } FunctionMATCH& FunctionMATCH::operator=(eckit::sql::expression::function::FunctionMATCH const&) { // TODO: NOTIMP; return *this; } void FunctionMATCH::collect(const std::vector& v) { subqueryResult_.push_back(v); } const type::SQLType* FunctionMATCH::type() const { return &type::SQLType::lookup("real"); } // TODO: bool? std::shared_ptr FunctionMATCH::clone() const { return std::make_shared(*this); } double FunctionMATCH::eval(bool& missing) const { std::vector vs(size_); for (size_t i(0); i < size_; ++i) { bool missing(false); vs[i] = args_[i]->eval(missing); } return std::binary_search(subqueryResult_.begin(), subqueryResult_.end(), vs); } } // namespace function } // namespace expression } // namespace sql } // namespace eckit eckit-2.0.7/src/eckit/sql/expression/function/FunctionTDIFF.cc0000664000175000017500000000367515161702250024377 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionTDIFF.h" #include "eckit/sql/SQLOutput.h" #include "eckit/types/DateTime.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder tdiffFunctionBuilder("tdiff"); FunctionTDIFF::FunctionTDIFF(const std::string& name, const expression::Expressions& args) : FunctionIntegerExpression(name, args) {} FunctionTDIFF::FunctionTDIFF(const FunctionTDIFF& other) : FunctionIntegerExpression(other.name_, other.args_) {} std::shared_ptr FunctionTDIFF::clone() const { return std::make_shared(*this); } FunctionTDIFF::~FunctionTDIFF() {} double FunctionTDIFF::eval(bool& missing) const { int indate = (int)args_[0]->eval(missing); int intime = (int)args_[1]->eval(missing); int andate = (int)args_[2]->eval(missing); int antime = (int)args_[3]->eval(missing); if (missing) { return 0; } // Check for invalid values try { eckit::Date d1(indate); eckit::Date d2(andate); eckit::Time t1(intime / 10000, (intime % 10000) / 100, intime % 100); eckit::Time t2(antime / 10000, (antime % 10000) / 100, antime % 100); eckit::DateTime dt1(d1, t1); eckit::DateTime dt2(d2, t2); return dt1 - dt2; } catch (BadValue& e) { missing = true; return 0; } } const eckit::sql::type::SQLType* FunctionTDIFF::type() const { return &eckit::sql::type::SQLType::lookup("integer"); } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionCOUNT.h0000664000175000017500000000313615161702250024265 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionCOUNT.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionCOUNT_H #define FunctionCOUNT_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionCOUNT : public FunctionExpression { public: FunctionCOUNT(const std::string&, const expression::Expressions&); FunctionCOUNT(const FunctionCOUNT&); ~FunctionCOUNT(); std::shared_ptr clone() const override; static int arity() { return 1; } static const char* help() { return "Average (aggregate function)"; } private: // No copy allowed FunctionCOUNT& operator=(const FunctionCOUNT&); unsigned long long count_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; using FunctionExpression::eval; double eval(bool& missing) const override; bool isAggregate() const override { return true; } // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionCOUNT& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionROWNUMBER.h0000664000175000017500000000343215161702250024754 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file FunctionROWNUMBER.h /// Piotr Kuchta - (C) ECMWF July 2009 #ifndef FunctionROWNUMBER_H #define FunctionROWNUMBER_H #include "eckit/sql/expression/function/FunctionIntegerExpression.h" namespace eckit::sql::expression::function { class FunctionROWNUMBER : public FunctionIntegerExpression { public: FunctionROWNUMBER(const std::string&, const expression::Expressions&); FunctionROWNUMBER(const FunctionROWNUMBER&); ~FunctionROWNUMBER(); // -- Overridden methods const eckit::sql::type::SQLType* type() const override; std::shared_ptr clone() const override; static int arity() { return 0; } protected: // -- Overridden methods void print(std::ostream& s) const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; bool isConstant() const override; void partialResult() override; using FunctionIntegerExpression::eval; double eval(bool& missing) const override; std::shared_ptr simplify(bool&) override; bool isAggregate() const override { return false; } private: // No copy allowed FunctionROWNUMBER& operator=(const FunctionROWNUMBER&); unsigned long long* count_; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionROWNUMBER& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionMAX.cc0000664000175000017500000000363315161702250024162 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/expression/function/FunctionMAX.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder maxFunctionBuilder("max"); FunctionMAX::FunctionMAX(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), value_(-DBL_MAX) {} FunctionMAX::FunctionMAX(const FunctionMAX& other) : FunctionExpression(other.name_, other.args_), value_(other.value_) {} std::shared_ptr FunctionMAX::clone() const { return std::make_shared(*this); } const eckit::sql::type::SQLType* FunctionMAX::type() const { return args_[0]->type(); } FunctionMAX::~FunctionMAX() {} double FunctionMAX::eval(bool& missing) const { if (value_ == -DBL_MAX) { missing = true; } return value_; } void FunctionMAX::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); value_ = -DBL_MAX; } void FunctionMAX::output(SQLOutput& s) const { bool missing = false; double d = eval(missing); type()->output(s, d, missing); } void FunctionMAX::cleanup(SQLSelect& sql) { FunctionExpression::cleanup(sql); value_ = -DBL_MAX; } void FunctionMAX::partialResult() { bool missing = false; double value = args_[0]->eval(missing); if (!missing) { if (value > value_) { value_ = value; } } } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionRMS.h0000664000175000017500000000306215161702250024034 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionRMS.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionRMS_H #define FunctionRMS_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionRMS : public FunctionExpression { public: FunctionRMS(const std::string&, const expression::Expressions&); FunctionRMS(const FunctionRMS&); ~FunctionRMS(); // -- Overridden methods using FunctionExpression::eval; double eval(bool& missing) const override; std::shared_ptr clone() const override; static int arity() { return 1; } private: // No copy allowed FunctionRMS& operator=(const FunctionRMS&); unsigned long long count_; double squares_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; bool isAggregate() const override { return true; } // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionRMS& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionJULIAN.cc0000664000175000017500000000330115161702250024507 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionJULIAN.h" #include "eckit/exception/Exceptions.h" #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/types/Date.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder julianFunctionBuilder("julian"); static FunctionBuilder jdFunctionBuilder("jd"); static FunctionBuilder julian_dateFunctionBuilder("julian_date"); FunctionJULIAN::FunctionJULIAN(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args) {} FunctionJULIAN::FunctionJULIAN(const FunctionJULIAN& other) : FunctionExpression(other.name_, other.args_) {} std::shared_ptr FunctionJULIAN::clone() const { return std::make_shared(*this); } FunctionJULIAN::~FunctionJULIAN() {} const type::SQLType* FunctionJULIAN::type() const { return &type::SQLType::lookup("real"); } double FunctionJULIAN::eval(bool& missing) const { int indate = (int)args_[0]->eval(missing); // int intime = (int) args_[1]->eval(missing); // unused // TODO: shold we return MISSING_VALUE_INT in case missing == true here? eckit::Date date(indate); return date.julian(); } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionOR.h0000664000175000017500000000262515161702250023717 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionOR.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionOR_H #define FunctionOR_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionOR : public FunctionExpression { public: FunctionOR(const std::string&, const expression::Expressions&); FunctionOR(const FunctionOR&); ~FunctionOR(); // -- Overridden methods std::shared_ptr clone() const override; using FunctionExpression::eval; double eval(bool& missing) const override; const eckit::sql::type::SQLType* type() const override; std::shared_ptr simplify(bool&) override; static int arity() { return 2; } static const char* help() { return ""; } private: // No copy allowed FunctionOR& operator=(const FunctionOR&); // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionOR& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionExpression.h0000664000175000017500000000432015161702250025530 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 2003 #ifndef eckit_sql_FunctionExpression_H #define eckit_sql_FunctionExpression_H #include "eckit/sql/expression/SQLExpressions.h" namespace eckit::sql::expression::function { //---------------------------------------------------------------------------------------------------------------------- class FunctionExpression : public SQLExpression { public: FunctionExpression(const std::string&, const expression::Expressions&); FunctionExpression(const FunctionExpression&); ~FunctionExpression(); void print(std::ostream& s) const override; void preprepare(SQLSelect& sql) override; void prepare(SQLSelect& sql) override; void updateType(SQLSelect& sql) override; void cleanup(SQLSelect& sql) override; bool isConstant() const override; std::shared_ptr simplify(bool&) override; // double eval() const override; bool isAggregate() const override; void partialResult() override; const type::SQLType* type() const override; std::shared_ptr reshift(int minColumnShift) const override; // For SQLSelectFactory (maybe it should just friend SQLSelectFactory). expression::Expressions& args() { return args_; } static const char* help() { return ""; } protected: std::string name_; expression::Expressions args_; // void print(std::ostream&) const override; // -- Overridden methods void tables(std::set&) override; private: FunctionExpression& operator=(const FunctionExpression&); // For use inside reshift() void shiftArgs(int minColumnShift); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionFactory.h0000664000175000017500000001253315161702250025005 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // @author Simon Smart // @date January 2019 // @note Based on // File FunctionFactory.h // Baudouin Raoult - ECMWF Dec 03 #ifndef eckit_sql_expression_FunctionFactory_H #define eckit_sql_expression_FunctionFactory_H #include #include #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { //---------------------------------------------------------------------------------------------------------------------- class FunctionBuilderBase; class FunctionFactory { public: // types struct FuncInfo { std::string name; int arity; std::string help; }; public: // methods static FunctionFactory& instance(); void enregister(const std::string& name, int arity_, const FunctionBuilderBase* builder); void deregister(const std::string& name, int arity_, const FunctionBuilderBase* builder); std::vector functionsInfo(); std::shared_ptr build(const std::string& name, const expression::Expressions& args) const; std::shared_ptr build(const std::string&, std::shared_ptr); std::shared_ptr build(const std::string&, std::shared_ptr, std::shared_ptr); std::shared_ptr build(const std::string&, std::shared_ptr, std::shared_ptr, std::shared_ptr); private: // methods FunctionFactory(); ~FunctionFactory(); private: // members mutable std::mutex m_; std::map, const FunctionBuilderBase*> builders_; }; //---------------------------------------------------------------------------------------------------------------------- class FunctionBuilderBase { std::string name_; int arity_; std::string help_; public: // methods FunctionBuilderBase(const std::string& name, int arity, const char* help); virtual ~FunctionBuilderBase(); virtual std::shared_ptr make(const std::string& name, const expression::Expressions& args) const = 0; int arity() const; std::string help() const; }; template class FunctionBuilder : public FunctionBuilderBase { std::shared_ptr make(const std::string& name, const expression::Expressions& args) const override { return std::make_shared(name, args); } public: // methods FunctionBuilder(const std::string& name, const char* help = nullptr) : FunctionBuilderBase(name, FunctionType::arity(), help ? help : FunctionType::help()) {} ~FunctionBuilder() override {} }; //---------------------------------------------------------------------------------------------------------------------- #if 0 class FunctionFactoryBase { protected: int arity_; std::string name_; std::string help_; virtual std::shared_ptr make(const std::string&,const expression::Expressions&) = 0; public: //FunctionFactoryBase() : name_("FunctionFactory"), arity_(-1) {} FunctionFactoryBase(const std::string& name, int arity, const std::string& help); ~FunctionFactoryBase() override; std::shared_ptr build(const std::string&, std::shared_ptr); std::shared_ptr build(const std::string&, std::shared_ptr, std::shared_ptr); std::shared_ptr build(const std::string&, std::shared_ptr, std::shared_ptr, std::shared_ptr); std::shared_ptr build(const std::string&, const expression::Expressions&); }; class FunctionFactory : public FunctionFactoryBase { public: static FunctionFactory& instance(); FunctionFactory(); // : FunctionFactoryBase("FunctionFactory", -1) {} ~FunctionFactory() override {} using FunctionInfo = std::vector, std::string>>; FunctionInfo& functionsInfo(); private: std::shared_ptr make(const std::string&,const expression::Expressions&) override { NOTIMP; return 0; } std::map, FunctionFactoryBase*> map_; FunctionInfo functionInfo_; }; template class FunctionMaker : public FunctionFactoryBase { std::shared_ptr make(const std::string& name, const expression::Expressions& args) override { return std::make_shared(name, args); } public: FunctionMaker(const std::string& name, int arity, const std::string& help) : FunctionFactoryBase(name, arity, help) {} }; #endif //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionNORM.cc0000664000175000017500000000370515161702250024310 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/expression/function/FunctionNORM.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder normFunctionBuilder("norm"); FunctionNORM::FunctionNORM(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), value_(0), resultNULL_(true) {} FunctionNORM::FunctionNORM(const FunctionNORM& other) : FunctionExpression(other.name_, other.args_), value_(other.value_), resultNULL_(other.resultNULL_) {} std::shared_ptr FunctionNORM::clone() const { return std::make_shared(*this); } FunctionNORM::~FunctionNORM() {} const type::SQLType* FunctionNORM::type() const { return &type::SQLType::lookup("double"); } double FunctionNORM::eval(bool& missing) const { if (resultNULL_) { missing = true; return (double)0; } double lvalue = (value_ > 0) ? sqrt(value_) : (double)0; return lvalue; } void FunctionNORM::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); value_ = 0; } void FunctionNORM::cleanup(SQLSelect& sql) { FunctionExpression::cleanup(sql); value_ = 0; } void FunctionNORM::partialResult() { bool missing1 = false; bool missing2 = false; double x = args_[0]->eval(missing1); double y = args_[1]->eval(missing2); if (!missing1 && !missing2) { value_ += x * y; resultNULL_ = false; } } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionMIN.h0000664000175000017500000000303715161702250024020 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionMIN.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionMIN_H #define FunctionMIN_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionMIN : public FunctionExpression { public: FunctionMIN(const std::string&, const expression::Expressions&); FunctionMIN(const FunctionMIN&); ~FunctionMIN(); std::shared_ptr clone() const override; static int arity() { return 1; } private: // No copy allowed FunctionMIN& operator=(const FunctionMIN&); double value_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; using FunctionExpression::eval; double eval(bool& missing) const override; bool isAggregate() const override { return true; } void output(SQLOutput&) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionMIN& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionSTDEV.h0000664000175000017500000000236415161702250024264 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionSTDEV.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionSTDEV_H #define FunctionSTDEV_H #include "eckit/sql/expression/function/FunctionVAR.h" namespace eckit::sql::expression::function { class FunctionSTDEV : public FunctionVAR { public: FunctionSTDEV(const std::string&, const expression::Expressions&); FunctionSTDEV(const FunctionSTDEV&); ~FunctionSTDEV(); std::shared_ptr clone() const override; private: // No copy allowed FunctionSTDEV& operator=(const FunctionSTDEV&); using FunctionVAR::eval; double eval(bool& missing) const override; const eckit::sql::type::SQLType* type() const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionSTDEV& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionRLIKE.cc0000664000175000017500000000576715161702250024415 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionRLIKE.h" #include "eckit/exception/Exceptions.h" #include "eckit/sql/expression/ColumnExpression.h" #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/type/SQLType.h" #include "eckit/utils/Regex.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder rlikeFunctionBuilder("rlike"); static FunctionBuilder likeFunctionBuilder("like"); using namespace eckit::sql::type; const type::SQLType* FunctionRLIKE::type() const { return &type::SQLType::lookup("double"); } FunctionRLIKE::FunctionRLIKE(const FunctionRLIKE& other) : FunctionExpression(other.name_, other.args_), re_(new eckit::Regex(*other.re_)) {} FunctionRLIKE::FunctionRLIKE(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), re_() {} std::shared_ptr FunctionRLIKE::clone() const { return std::make_shared(*this); } FunctionRLIKE::~FunctionRLIKE() {} void FunctionRLIKE::trimStringInDouble(char*& p, size_t& len) { len = 0; for (; len < sizeof(double) && isprint(p[len]); ++len) { ; } for (; len > 0 && isspace(p[len - 1]); --len) { ; } size_t plen = len; for (char* pp = p; isspace(*p) && p < pp + plen;) { ++p; --len; } } void FunctionRLIKE::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); SQLExpression &l(*args_[0]), &r(*args_[1]); if (l.type()->getKind() != SQLType::stringType || r.type()->getKind() != SQLType::stringType) { throw eckit::UserError("Arguments of RLIKE must be of string type"); } bool missing(false); double v2(r.eval(missing)); char* p2 = reinterpret_cast(&v2); size_t len2(sizeof(double)); trimStringInDouble(p2, len2); std::string re(p2, len2); // eckit::Log::info() << "FunctionRLIKE::prepare: regex: '" << re << "'" << std::endl; re_.reset(new eckit::Regex(re)); } bool FunctionRLIKE::match(const SQLExpression& l, const SQLExpression& r, bool& missing) const { double v1 = l.eval(missing); if (missing) { return false; } char* p1(reinterpret_cast(&v1)); size_t len1(sizeof(double)); trimStringInDouble(p1, len1); std::string s1(p1, len1); bool ret = re_->match(s1); // eckit::Log::info() << "FunctionRLIKE::match '" << s1 << "' => " << ret << std::endl; return ret; } double FunctionRLIKE::eval(bool& missing) const { return match(*args_[0], *args_[1], missing); } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionAND.cc0000664000175000017500000000451115161702250024133 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionAND.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder andFunctionBuilder("and"); FunctionAND::FunctionAND(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args) {} FunctionAND::FunctionAND(const FunctionAND& other) : FunctionExpression(other.name_, other.args_) {} std::shared_ptr FunctionAND::clone() const { return std::make_shared(*this); } FunctionAND::~FunctionAND() {} const type::SQLType* FunctionAND::type() const { return &type::SQLType::lookup("double"); } // TODO: bool? double FunctionAND::eval(bool& missing) const { return args_[0]->eval(missing) && args_[1]->eval(missing); } bool FunctionAND::andSplit(expression::Expressions& e) { bool ok = false; if (!args_[0]->andSplit(e)) { e.push_back(args_[0]); ok = true; } if (!args_[1]->andSplit(e)) { e.push_back(args_[1]); ok = true; } return ok; } std::shared_ptr FunctionAND::simplify(bool& changed) { std::shared_ptr x = FunctionExpression::simplify(changed); if (x) { return x; } for (int i = 0; i < 2; i++) { bool missing = false; if (args_[i]->isConstant()) { if (args_[i]->eval(missing)) { std::cout << "SYMPLIFY " << *this << " to "; changed = true; std::shared_ptr x = args_[1 - i]; args_.clear(); std::cout << *x << std::endl; return x; } std::cout << "SYMPLIFY " << *this << "to 0 " << std::endl; changed = true; return SQLExpression::number(0); } } return nullptr; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionNVL.cc0000664000175000017500000000252515161702250024173 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionNVL.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder nvlFunctionBuilder("nvl"); FunctionNVL::FunctionNVL(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args) {} FunctionNVL::FunctionNVL(const FunctionNVL& other) : FunctionExpression(other.name_, other.args_) {} FunctionNVL::~FunctionNVL() {} std::shared_ptr FunctionNVL::clone() const { return std::make_shared(*this); } const type::SQLType* FunctionNVL::type() const { return &type::SQLType::lookup("real"); } // Don't set the missing flag double FunctionNVL::eval(bool&) const { bool missing = false; double x = args_[0]->eval(missing); return missing ? args_[1]->eval(missing) : x; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionJULIAN_SECONDS.h0000664000175000017500000000304215161702250025531 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file FunctionJULIAN_SECONDS.h /// ECMWF February 2014 #ifndef FunctionJULIAN_SECONDS_H #define FunctionJULIAN_SECONDS_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionJULIAN_SECONDS : public FunctionExpression { public: FunctionJULIAN_SECONDS(const std::string&, const expression::Expressions&); FunctionJULIAN_SECONDS(const FunctionJULIAN_SECONDS&); ~FunctionJULIAN_SECONDS(); // Change to virtual if base class std::shared_ptr clone() const override; static int arity() { return 2; } static const char* help() { return "Returns time in Julian calendar expressed in seconds"; } private: // No copy allowed FunctionJULIAN_SECONDS& operator=(const FunctionJULIAN_SECONDS&); // -- Overridden methods const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend ostream& operator<<(ostream& s,const FunctionJULIAN_SECONDS& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionTHIN.h0000664000175000017500000000325515161702250024141 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file FunctionTHIN.h /// ECMWF July 2010 #ifndef FunctionTHIN_H #define FunctionTHIN_H #include "eckit/sql/expression/function/FunctionIntegerExpression.h" namespace eckit::sql::expression::function { class FunctionTHIN : public FunctionIntegerExpression { public: FunctionTHIN(const std::string&, const expression::Expressions&); FunctionTHIN(const FunctionTHIN&); ~FunctionTHIN(); std::shared_ptr clone() const override; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; static int arity() { return 2; } protected: // -- Overridden methods void print(std::ostream& s) const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; bool isConstant() const override; using FunctionIntegerExpression::eval; double eval(bool& missing) const override; std::shared_ptr simplify(bool&) override; bool isAggregate() const override { return false; } private: // No copy allowed FunctionTHIN& operator=(const FunctionTHIN&); unsigned long long* count_; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionTHIN& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionJULIAN.h0000664000175000017500000000251015161702250024352 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionJULIAN.h // ECMWF July 2010 #ifndef FunctionJULIAN_H #define FunctionJULIAN_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionJULIAN : public FunctionExpression { public: FunctionJULIAN(const std::string&, const expression::Expressions&); FunctionJULIAN(const FunctionJULIAN&); ~FunctionJULIAN(); std::shared_ptr clone() const override; static int arity() { return 2; } private: // No copy allowed FunctionJULIAN& operator=(const FunctionJULIAN&); // -- Overridden methods const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionJULIAN& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionNOT_NULL.h0000664000175000017500000000255515161702250024673 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionNOT_NULL.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionNOT_NULL_H #define FunctionNOT_NULL_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionNOT_NULL : public FunctionExpression { public: FunctionNOT_NULL(const std::string&, const expression::Expressions&); FunctionNOT_NULL(const FunctionNOT_NULL&); ~FunctionNOT_NULL(); std::shared_ptr clone() const override; static int arity() { return 1; } private: // No copy allowed const eckit::sql::type::SQLType* type() const override; FunctionNOT_NULL& operator=(const FunctionNOT_NULL&); // -- Overridden methods using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionNOT_NULL& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionLAST.h0000664000175000017500000000317315161702250024141 0ustar alastairalastair/* * (C) Copyright 1996-2016 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionLAST.h // Piotr Kuchta - ECMWF Nov 2016 #ifndef eckit_sql_function_FunctionLAST_H #define eckit_sql_function_FunctionLAST_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionLAST : public FunctionExpression { public: FunctionLAST(const std::string&, const expression::Expressions&); FunctionLAST(const FunctionLAST&); ~FunctionLAST(); std::shared_ptr clone() const override; static int arity() { return 1; } static const char* help() { return ""; } private: // No copy allowed FunctionLAST& operator=(const FunctionLAST&); double value_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; using FunctionExpression::eval; double eval(bool& missing) const override; bool isAggregate() const override { return true; } void output(SQLOutput&) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionLAST& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionROWNUMBER.cc0000664000175000017500000000367415161702250025122 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file FunctionROWNUMBER.h /// Piotr Kuchta - (C) ECMWF July 2009 #include "eckit/sql/expression/function/FunctionROWNUMBER.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/SQLTable.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder rownumberFunctionBuilder("rownumber"); FunctionROWNUMBER::FunctionROWNUMBER(const std::string& name, const expression::Expressions& args) : FunctionIntegerExpression(name, args), count_{nullptr} {} FunctionROWNUMBER::FunctionROWNUMBER(const FunctionROWNUMBER& other) : FunctionIntegerExpression(other.name_, other.args_), count_(other.count_) {} std::shared_ptr FunctionROWNUMBER::clone() const { return std::make_shared(*this); } FunctionROWNUMBER::~FunctionROWNUMBER() {} void FunctionROWNUMBER::print(std::ostream& s) const { s << "rownumber()"; } double FunctionROWNUMBER::eval(bool& missing) const { return *count_; } void FunctionROWNUMBER::prepare(SQLSelect& sql) { count_ = &sql.total_; } void FunctionROWNUMBER::cleanup(SQLSelect& sql) {} bool FunctionROWNUMBER::isConstant() const { return false; } std::shared_ptr FunctionROWNUMBER::simplify(bool&) { return nullptr; } void FunctionROWNUMBER::partialResult() { /*NOTIMP;*/ } const eckit::sql::type::SQLType* FunctionROWNUMBER::type() const { return &eckit::sql::type::SQLType::lookup("integer"); } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionNULL.h0000664000175000017500000000250015161702250024141 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionNULL.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionNULL_H #define FunctionNULL_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionNULL : public FunctionExpression { public: FunctionNULL(const std::string&, const expression::Expressions&); FunctionNULL(const FunctionNULL&); ~FunctionNULL(); std::shared_ptr clone() const override; static int arity() { return 1; } private: // No copy allowed const eckit::sql::type::SQLType* type() const override; FunctionNULL& operator=(const FunctionNULL&); // -- Overridden methods using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionNULL& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionEQ.h0000664000175000017500000000301415161702250023675 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionEQ.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionEQ_H #define FunctionEQ_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionEQ : public FunctionExpression { public: FunctionEQ(const std::string&, const expression::Expressions&); FunctionEQ(const FunctionEQ&); ~FunctionEQ(); static bool equal(const SQLExpression& l, const SQLExpression& r, bool& missing); static void trimStringInDouble(char*& p, size_t& len); std::shared_ptr clone() const override; static int arity() { return 2; } private: // No copy allowed FunctionEQ& operator=(const FunctionEQ&); double tmp_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; std::shared_ptr simplify(bool&) override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionEQ& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionVAR.cc0000664000175000017500000000421315161702250024160 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionVAR.h" #include "eckit/sql/expression/function/FunctionFactory.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder varFunctionBuilder("var"); static FunctionBuilder varpFunctionBuilder("varp"); const type::SQLType* FunctionVAR::type() const { return &type::SQLType::lookup("double"); } FunctionVAR::FunctionVAR(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), count_(0), value_(0), squares_(0) {} FunctionVAR::FunctionVAR(const FunctionVAR& other) : FunctionExpression(other.name_, other.args_), count_(other.count_), value_(other.value_), squares_(other.squares_) {} FunctionVAR::~FunctionVAR() {} std::shared_ptr FunctionVAR::clone() const { return std::make_shared(*this); } double FunctionVAR::eval(bool& missing) const { double x, y; if (!count_) { missing = true; return 0; } x = value_ / count_; y = squares_ / count_; return y - x * x; } void FunctionVAR::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); value_ = 0; count_ = 0; squares_ = 0; } void FunctionVAR::cleanup(SQLSelect& sql) { // cout << "Cleanup FunctionVAR " << count_ << " " << value_ << std::endl; FunctionExpression::cleanup(sql); squares_ = 0; value_ = 0; count_ = 0; } void FunctionVAR::partialResult() { bool missing = false; double value = args_[0]->eval(missing); if (!missing) { value_ += value; squares_ += value * value; count_++; } // else cout << "missing" << std::endl; } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionNORM.h0000664000175000017500000000300415161702250024142 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionNORM.h // ECMWF July 2010 #ifndef FunctionNORM_H #define FunctionNORM_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionNORM : public FunctionExpression { public: FunctionNORM(const std::string&, const expression::Expressions&); FunctionNORM(const FunctionNORM&); ~FunctionNORM(); std::shared_ptr clone() const override; static int arity() { return 2; } private: // No copy allowed FunctionNORM& operator=(const FunctionNORM&); double value_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; void partialResult() override; using FunctionExpression::eval; double eval(bool& missing) const override; bool isAggregate() const override { return true; } bool resultNULL_; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionNORM& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionNOT_IN.h0000664000175000017500000000255015161702250024422 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionNOT_IN.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionNOT_IN_H #define FunctionNOT_IN_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionNOT_IN : public FunctionExpression { public: FunctionNOT_IN(const std::string&, const expression::Expressions&); FunctionNOT_IN(const FunctionNOT_IN&); ~FunctionNOT_IN(); std::shared_ptr clone() const override; static int arity() { return -1; } private: // No copy allowed FunctionNOT_IN& operator=(const FunctionNOT_IN&); int size_; // -- Overridden methods const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionNOT_IN& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionIN.h0000664000175000017500000000244215161702250023702 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File FunctionIN.h // Baudouin Raoult - ECMWF Dec 03 #ifndef FunctionIN_H #define FunctionIN_H #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql::expression::function { class FunctionIN : public FunctionExpression { public: FunctionIN(const std::string&, const expression::Expressions&); FunctionIN(const FunctionIN&); ~FunctionIN(); std::shared_ptr clone() const override; static int arity() { return -1; } private: // No copy allowed FunctionIN& operator=(const FunctionIN&); size_t size_; const eckit::sql::type::SQLType* type() const override; using FunctionExpression::eval; double eval(bool& missing) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const FunctionIN& p) // { p.print(s); return s; } }; } // namespace eckit::sql::expression::function #endif eckit-2.0.7/src/eckit/sql/expression/function/FunctionSTDEV.cc0000664000175000017500000000275015161702250024421 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/expression/function/FunctionSTDEV.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder stdevFunctionBuilder("stdev"); static FunctionBuilder stddevFunctionBuilder("stddev"); static FunctionBuilder stdevpFunctionBuilder("stdevp"); FunctionSTDEV::FunctionSTDEV(const std::string& name, const expression::Expressions& args) : FunctionVAR(name, args) {} FunctionSTDEV::FunctionSTDEV(const FunctionSTDEV& other) : FunctionVAR(static_cast(other)) {} FunctionSTDEV::~FunctionSTDEV() {} std::shared_ptr FunctionSTDEV::clone() const { return std::make_shared(*this); } const type::SQLType* FunctionSTDEV::type() const { return &type::SQLType::lookup("double"); } double FunctionSTDEV::eval(bool& missing) const { double x = FunctionVAR::eval(missing); if (x < 0) { x = 0; } return sqrt(x); } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/DoubleFunctions.cc0000664000175000017500000003665315161702250025142 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/expression/function/FunctionFactory.h" #include #include #include namespace eckit::sql::expression::function { //---------------------------------------------------------------------------------------------------------------------- template class ArityFunction : public FunctionExpression { std::shared_ptr clone() const { return std::make_shared(name_, args_); } public: using FunctionExpression::FunctionExpression; static int arity() { return ARITY; } }; template class UnaryFunction : public ArityFunction, 1> { using FunctionExpression::eval; double eval(bool& m) const { double a0 = this->args_[0]->eval(m); if (m) { return this->missingValue_; } return FN(a0); } public: using ArityFunction, 1>::ArityFunction; }; template class BinaryFunction : public ArityFunction, 2> { using FunctionExpression::eval; double eval(bool& m) const { double a0 = this->args_[0]->eval(m); if (m) { return this->missingValue_; } double a1 = this->args_[1]->eval(m); if (m) { return this->missingValue_; } return FN(a0, a1); } public: using ArityFunction, 2>::ArityFunction; }; template class TertiaryFunction : public ArityFunction, 3> { using FunctionExpression::eval; double eval(bool& m) const { double a0 = this->args_[0]->eval(m); if (m) { return this->missingValue_; } double a1 = this->args_[1]->eval(m); if (m) { return this->missingValue_; } double a2 = this->args_[2]->eval(m); if (m) { return this->missingValue_; } return FN(a0, a1, a2); } public: using ArityFunction, 3>::ArityFunction; }; template class QuaternaryFunction : public ArityFunction, 4> { using FunctionExpression::eval; double eval(bool& m) const { double a0 = this->args_[0]->eval(m); if (m) { return this->missingValue_; } double a1 = this->args_[1]->eval(m); if (m) { return this->missingValue_; } double a2 = this->args_[2]->eval(m); if (m) { return this->missingValue_; } double a3 = this->args_[3]->eval(m); if (m) { return this->missingValue_; } return FN(a0, a1, a2, a3); } public: using ArityFunction, 4>::ArityFunction; }; template class QuinaryFunction : public ArityFunction, 5> { using FunctionExpression::eval; double eval(bool& m) const { double a0 = this->args_[0]->eval(m); if (m) { return this->missingValue_; } double a1 = this->args_[1]->eval(m); if (m) { return this->missingValue_; } double a2 = this->args_[2]->eval(m); if (m) { return this->missingValue_; } double a3 = this->args_[3]->eval(m); if (m) { return this->missingValue_; } double a4 = this->args_[4]->eval(m); if (m) { return this->missingValue_; } return FN(a0, a1, a2, a3, a4); } public: using ArityFunction, 5>::ArityFunction; }; //---------------------------------------------------------------------------------------------------------------------- const double R_Earth_km = 180 * 60 / M_PI * 1.852; const double R_Earth = 180 * 60 / M_PI * 1.852 * 1000.0; const double D2R = M_PI / 180.0; const double R2D = 180.0 / M_PI; inline double abs(double x) { return fabs(x); } // Note: ODB's trigonometric funcs require args in degrees // and return degrees (where applicable) inline double Func_acos(double x) { return (R2D * acos(x)); } inline double Func_asin(double x) { return (R2D * asin(x)); } inline double Func_atan(double x) { return (R2D * atan(x)); } inline double Func_atan2(double x, double y) { return (R2D * atan2(x, y)); } inline double Func_cos(double x) { return (cos(D2R * x)); } inline double Func_sin(double x) { return (sin(D2R * x)); } inline double Func_tan(double x) { return (tan(D2R * x)); } inline double mod(double x, double y) { return fmod(x, y); } inline double Func_pow(double x, double y) { return ((y) == 2 ? (x) * (x) : pow(x, y)); } inline double ln(double x) { return log(x); } inline double lg(double x) { return log10(x); } inline double twice(double x) { return 2 * x; } const double ZERO_POINT = ((double)273.15e0); inline double celsius(double x) { return x - ZERO_POINT; } inline double k2c(double x) { return x - ZERO_POINT; } inline double kelvin(double x) { return x + ZERO_POINT; } inline double c2k(double x) { return x + ZERO_POINT; } inline double c2f(double x) { return ((9 * x) / 5) + 32; } inline double f2c(double x) { return ((x - 32) * 5) / 9; } inline double f2k(double x) { return c2k(f2c(x)); } inline double k2f(double x) { return c2f(k2c(x)); } inline double fahrenheit(double x) { return c2f(k2c(x)); } inline double radians(double x) { return x * D2R; } inline double deg2rad(double x) { return x * D2R; } inline double degrees(double x) { return x * R2D; } inline double rad2deg(double x) { return x * R2D; } inline double speed(double u, double v) { return sqrt(u * u + v * v); } inline double ff(double u, double v) { return speed(u, v); } inline double direction(double u, double v) { return fmod(Func_atan2(-u, -v) + 360., 360.); } /// Distance in meters. inline double distance(double lat1, double lon1, double lat2, double lon2) { return R_Earth * acos(Func_sin(lat1) * Func_sin(lat2) + Func_cos(lat1) * Func_cos(lat2) * Func_cos(lon1 - lon2)); } inline double km(double x) { return R_Earth_km * x; } /// in kilometers inline double km(double lat1, double lon1, double lat2, double lon2) { return R_Earth_km * acos(Func_sin(lat1) * Func_sin(lat2) + Func_cos(lat1) * Func_cos(lat2) * Func_cos(lon1 - lon2)); } /// in kilometers inline double dist(double reflat, double reflon, double refdist_km, double obslat, double obslon) { return (double)(R_Earth_km * acos(Func_cos(reflat) * Func_cos(obslat) * Func_cos(obslon - reflon) + Func_sin(reflat) * Func_sin(obslat)) <= (refdist_km)); } inline double circle(double x, double x0, double y, double y0, double r) { return (Func_pow(x - x0, 2) + Func_pow(y - y0, 2) <= Func_pow(r, 2)); } inline double rad(double reflat, double reflon, double refdeg, double obslat, double obslon) { // double v (Func_cos(reflat) * Func_cos(obslat) * Func_cos(obslon-reflon) + Func_sin(reflat) * Func_sin(obslat) // ); // int digs ( 3 + DBL_MANT_DIG - DBL_MIN_EXP ); return (acos(Func_cos(reflat) * Func_cos(obslat) * Func_cos(obslon - reflon) + Func_sin(reflat) * Func_sin(obslat)) <= D2R * refdeg) ? 1.0 : 0.0; } inline double between(double x, double a, double b) { return x >= a && x <= b; } inline double not_between(double x, double a, double b) { return x < a || x > b; } inline double between_exclude_first(double x, double a, double b) { return x > a && x <= b; } inline double between_exclude_second(double x, double a, double b) { return x >= a && x < b; } inline double between_exclude_both(double x, double a, double b) { return x > a && x < b; } double ibits(double X, double Pos, double Len) { constexpr int MAXBITS = 32; int rc = 0; /* the default */ X = trunc(X); Pos = trunc(Pos); Len = trunc(Len); if (X >= INT_MIN && X <= INT_MAX && Pos >= 0 && Pos < MAXBITS && Len >= 1 && Len <= MAXBITS) { rc = (int(X) >> int(Pos)) & ((1 << int(Len)) - 1); } return (double)rc; } double negate_double(double n) { return -n; } double logical_not_double(double n) { return !n; } double greater_double(double l, double r) { return l > r; } double greater_equal_double(double l, double r) { return l >= r; } double less_double(double l, double r) { return l < r; } double less_equal_double(double l, double r) { return l <= r; } double plus_double(double l, double r) { return l + r; } double minus_double(double l, double r) { return l - r; } double divides_double(double l, double r) { return l / r; } double ldexp_double(double l, double r) { return ldexp(l, r); } /// Multiplication is a special case class MultiplyFunction : public ArityFunction { using FunctionExpression::eval; double eval(bool& m) const { bool m0 = false; bool m1 = false; double a0 = args_[0]->eval(m0); double a1 = args_[1]->eval(m1); if ((a0 == 0 || a1 == 0) && !(m0 && m1)) { return 0; } if (m0 || m1) { m = true; return this->missingValue_; } return a0 * a1; } public: using ArityFunction::ArityFunction; }; //---------------------------------------------------------------------------------------------------------------------- // Static factories static FunctionBuilder> absBuilder("abs", "absolute value"); static FunctionBuilder> fabsBuilder("fabs", "absolute value"); static FunctionBuilder> acosBuilder("acos", "arc cosine"); static FunctionBuilder> asinBuilder("asin", "arc sine"); static FunctionBuilder> atanBuilder("atan", "arc tangent of one variable"); static FunctionBuilder> atan2Builder("atan2", "arc tangent of param1/param2 (y/x)"); static FunctionBuilder> cosBuilder("cos", "cosine"); static FunctionBuilder> sinBuilder("sin", "sine"); static FunctionBuilder> tanBuilder("tan", "tangent"); static FunctionBuilder> expBuilder("exp", "base-e exponential function, e raised to x"); static FunctionBuilder> coshBuilder("cosh", "hyperbolic cosine"); static FunctionBuilder> sinhBuilder("sinh", "hyperbolic sine"); static FunctionBuilder> tanhBuilder("tanh", "hyperbolic tangent"); static FunctionBuilder> logBuilder("log", "natural logarithm"); static FunctionBuilder> log10Builder("log10", "base-10 logarithm"); static FunctionBuilder> sqrtBuilder("sqrt", "square root"); static FunctionBuilder> modBuilder("mod", ""); static FunctionBuilder> fmodBuilder("fmod", ""); static FunctionBuilder> powBuilder("pow", "x ^ y"); static FunctionBuilder> lnBuilder("ln", "log base e"); static FunctionBuilder> lgBuilder("lg", "log base 10"); static FunctionBuilder> twiceBuilder("twice", ""); static FunctionBuilder> celsiusBuilder("celsius", ""); static FunctionBuilder> k2cBuilder("k2c", ""); static FunctionBuilder> kelvinBuilder("kelvin", ""); static FunctionBuilder> c2kBuilder("c2k", ""); static FunctionBuilder> c2fBuilder("c2f", ""); static FunctionBuilder> f2cBuilder("f2c", ""); static FunctionBuilder> f2kBuilder("f2k", ""); static FunctionBuilder> k2fBuilder("k2f", ""); static FunctionBuilder> fahrenheitBuilder("fahrenheit", ""); static FunctionBuilder> radiansBuilder("radians", ""); static FunctionBuilder> deg2radBuilder("deg2rad", ""); static FunctionBuilder> degreesBuilder("degrees", ""); static FunctionBuilder> rad2degBuilder("rad2deg", ""); static FunctionBuilder> speedBuilder("speed", ""); static FunctionBuilder> ffBuilder("ff", ""); static FunctionBuilder> directionBuilder("direction", ""); static FunctionBuilder> ddBuilder("dd", "direction"); static FunctionBuilder> dirBuilder("dir", "direction"); static FunctionBuilder> distance4Builder("distance", ""); static FunctionBuilder> kmBuilder("km", ""); static FunctionBuilder> km4Builder("km", ""); static FunctionBuilder> distBuilder("dist", ""); static FunctionBuilder> circleBuilder("circle", ""); static FunctionBuilder> rad5Builder("rad", ""); static FunctionBuilder> betweenBuilder("between", ""); static FunctionBuilder> not_betweenBuilder("not_between", ""); static FunctionBuilder> between_exclude_firstBuilder("between_exclude_first", ""); static FunctionBuilder> between_exclude_secondBuilder("between_exclude_second", ""); static FunctionBuilder> between_exclude_bothBuilder("between_exclude_both", ""); static FunctionBuilder> ibitsBuilder("ibits", ""); static FunctionBuilder> negate_doubleBuilder("-", ""); static FunctionBuilder> logical_not_doubleBuilder("not", "logical not"); static FunctionBuilder> greater_doubleBuilder(">", "greater than"); static FunctionBuilder> greater_equal_doubleBuilder(">=", "greater or equal"); static FunctionBuilder> less_doubleBuilder("<", "less than"); static FunctionBuilder> less_equal_doubleBuilder("<=", "less or equal"); static FunctionBuilder> plus_doubleBuilder("+", "add"); static FunctionBuilder> minus_doubleBuilder("-", "subtract"); static FunctionBuilder> divides_doubleBuilder("/", "divide"); static FunctionBuilder> ldexp_doubleBuilder("ldexp_double", ""); static FunctionBuilder multiplyBuilder("*", "multiply"); //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/function/FunctionLAST.cc0000664000175000017500000000350115161702250024272 0ustar alastairalastair/* * (C) Copyright 1996-2016 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/sql/expression/function/FunctionFactory.h" #include "eckit/sql/expression/function/FunctionLAST.h" namespace eckit::sql::expression::function { /* Static self-registration */ static FunctionBuilder lastFunctionBuilder("last"); FunctionLAST::FunctionLAST(const std::string& name, const expression::Expressions& args) : FunctionExpression(name, args), value_(DBL_MAX) {} FunctionLAST::FunctionLAST(const FunctionLAST& other) : FunctionExpression(other.name_, other.args_), value_(other.value_) {} std::shared_ptr FunctionLAST::clone() const { return std::make_shared(*this); } const eckit::sql::type::SQLType* FunctionLAST::type() const { return args_[0]->type(); } FunctionLAST::~FunctionLAST() {} double FunctionLAST::eval(bool& missing) const { if (value_ == DBL_MAX) { missing = true; } return value_; } void FunctionLAST::prepare(SQLSelect& sql) { FunctionExpression::prepare(sql); value_ = DBL_MAX; } void FunctionLAST::cleanup(SQLSelect& sql) { FunctionExpression::cleanup(sql); value_ = DBL_MAX; } void FunctionLAST::output(SQLOutput& s) const { bool missing(false); double d(eval(missing)); type()->output(s, d, missing); } void FunctionLAST::partialResult() { bool missing(false); value_ = (args_[0]->eval(missing)); } } // namespace eckit::sql::expression::function eckit-2.0.7/src/eckit/sql/expression/ColumnExpression.h0000664000175000017500000000646715161702250023371 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 2003 #ifndef eckit_sql_ColumnExpression_H #define eckit_sql_ColumnExpression_H #include "eckit/sql/expression/SQLExpression.h" namespace eckit::sql { class SQLOutput; class SQLTable; class SQLColumn; namespace expression { //---------------------------------------------------------------------------------------------------------------------- class ColumnExpression : public SQLExpression { public: ColumnExpression(const std::string&, const SQLTable*, int begin = -1, int end = -1); ColumnExpression(const std::string&, const std::string& tableReference, int begin = -1, int end = -1); ColumnExpression(const ColumnExpression&) = default; ~ColumnExpression(); const SQLTable* table() { return table_; } const double* current() { return value_->first; } std::shared_ptr clone() const override; std::shared_ptr reshift(int minColumnShift) const override; SQLExpression* nominalShift(int n) { nominalShift_ = n; return this; } protected: const type::SQLType* type_; // non-owning std::pair* value_; std::string columnName_; const SQLTable* table_; std::string tableReference_; std::string fullName_; int beginIndex_; int endIndex_; int nominalShift_; // -- Overridden methods void print(std::ostream& s) const override; void preprepare(SQLSelect& sql) override; void prepare(SQLSelect& sql) override; void updateType(SQLSelect& sql) override; void cleanup(SQLSelect& sql) override; double eval(bool& missing) const override; void eval(double* out, bool& missing) const override; std::string evalAsString(bool& missing) const override; bool isConstant() const override { return false; } void output(SQLOutput& s) const override; // We want to be able to construct a fully qualified column name, given a shortened one // (e.g. varno --> varno@body). But this needs to be overridable for BitExpressions // (e.g. datum_state.active --> datum_state.active@body even though the table column // might only be datum_state@body. i.e. we extract the bit subcolumns) virtual std::string tableColumnToFullname(const SQLColumn& column) const; private: ColumnExpression& operator=(const ColumnExpression&); // -- Overridden methods const type::SQLType* type() const override { return type_; } void expandStars(const std::vector>&, Expressions&) override; void tables(std::set&) override; friend class SQLSelectFactory; // friend std::ostream& operator<<(std::ostream& s,const ColumnExpression& p) // { p.print(s); return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace expression } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLIteratorOutput.h0000664000175000017500000000354715161702250021243 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file SQLIteratorOutput.h /// Piotr Kuchta - ECMWF Feb 09 #ifndef SQLIteratorOutput_H #define SQLIteratorOutput_H #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLOutput.h" namespace eckit { class SelectIterator; namespace sql { class ReaderIterator; template class SQLIteratorOutput : public SQLOutput { public: SQLIteratorOutput(T&); ~SQLIteratorOutput() override; protected: void print(std::ostream&) const override; private: // No copy allowed SQLIteratorOutput(const SQLIteratorOutput&); SQLIteratorOutput& operator=(const SQLIteratorOutput&); // -- Members T& iterator_; // bool headerSaved = true; unsigned long long count_; // -- Methods // None // -- Overridden methods virtual void size(int); virtual void reset(); void flush() override; virtual bool output(const expression::Expressions&); void prepare(SQLSelect&) override; void cleanup(SQLSelect&) override; virtual unsigned long long count(); virtual void outputReal(double, bool) { NOTIMP; }; virtual void outputDouble(double, bool) { NOTIMP; }; virtual void outputInt(double, bool) { NOTIMP; }; virtual void outputUnsignedInt(double, bool) { NOTIMP; }; virtual void outputString(double, bool) { NOTIMP; }; virtual void outputBitfield(double, bool) { NOTIMP; }; }; } // namespace sql } // namespace eckit #include "eckit/sql/SQLIteratorOutput.cc" #endif eckit-2.0.7/src/eckit/sql/SQLSelectFactory.cc0000664000175000017500000002263215161702250021132 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/types/Types.h" #include "eckit/utils/Translator.h" #include "eckit/sql/SQLDistinctOutput.h" #include "eckit/sql/SQLOrderOutput.h" #include "eckit/sql/SQLOutputConfig.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/SQLSelectFactory.h" #include "eckit/sql/SQLSession.h" #include "eckit/sql/expression/BitColumnExpression.h" #include "eckit/sql/expression/SQLExpressions.h" #include "eckit/sql/expression/ShiftedColumnExpression.h" #include "eckit/sql/expression/function/FunctionExpression.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SQLSelectFactory::SQLSelectFactory(SQLSession& session) : session_(session), database_(session.currentDatabase()), maxColumnShift_(0), minColumnShift_(0) {} /*void SQLSelectFactory::reset() { // TODO> we may need to delete things here... config_ = SQLOutputConfig::defaultConfig(); maxColumnShift_ = 0; minColumnShift_ = 0; } */ std::string SQLSelectFactory::index(const std::string& columnName, const SQLExpression* index) { if (index == nullptr) { return columnName; } bool missing = false; std::string idx = Translator()(int(index->eval(missing))); ASSERT(!missing); return columnName + "_" + idx; } std::shared_ptr SQLSelectFactory::createColumn(const std::string& columnName, const std::string& bitfieldName, std::shared_ptr& vectorIndex, const std::string& tableReference, std::shared_ptr& pshift) { if (!pshift->isConstant()) { throw eckit::UserError("Value of shift operator must be constant"); } bool missing = false; // Internally shift is an index in the cyclic buffer of old values, so the shift value is negative. int shift = -pshift->eval(missing); if (shift > maxColumnShift_) { maxColumnShift_ = shift; } if (shift < minColumnShift_) { minColumnShift_ = shift; } std::string expandedColumnName(index(columnName, vectorIndex.get())); // TODO: handle . return bitfieldName.size() ? (shift == 0 ? std::make_shared(expandedColumnName, bitfieldName, tableReference) : std::make_shared>( expandedColumnName, bitfieldName, tableReference, shift, -shift)) : (shift == 0 ? std::make_shared(expandedColumnName + tableReference, tableReference) : std::make_shared>( expandedColumnName + tableReference, tableReference, shift, -shift)); } /* SchemaAnalyzer& SQLSelectFactory::analyzer() { return SQLSession::current().currentDatabase().schemaAnalyzer(); } MetaData SQLSelectFactory::columns(const std::string& tableName) { const TableDef& tabledef ( enalyzer().findTable(tableName) ); const ColumnDefs& columnDefs ( tabledef.columns() ); //TODO: Convert ColumnDefs (from tabledef) into MetaData and push it into the SQLODAOutput ASSERT( false ); /// @todo this code must be fixed and return } */ SQLSelect* SQLSelectFactory::create(bool distinct, const Expressions& select_list, const std::string& into, const std::vector>& from, std::shared_ptr where, const Expressions& group_by, std::pair> order_by) { std::ostream& L(Log::debug()); if (where) { L << "SQLSelectFactory::create: where = " << *where << std::endl; } SQLSelect* r = nullptr; // Which tables are we selecting from? std::vector> fromTables; if (!from.size()) { L << "No from clause in SQL statement. Using implicit tables" << std::endl; } for (const SQLTable& tbl : from.size() ? from : database_.implicitTables()) { fromTables.push_back(tbl); } // n.b. it is acceptable for fromTables.size() == 0, if the expressions // are all constant. So we defer this check to later. // Expand any wildcards in the select statement Expressions select; for (Expressions::size_type i(0); i < select_list.size(); ++i) { L << "expandStars: " << *select_list[i] << std::endl; select_list[i]->expandStars(fromTables, select); } ASSERT(maxColumnShift_ >= 0); ASSERT(minColumnShift_ <= 0); if (minColumnShift_ < 0) { L << std::endl << "SELECT_LIST before reshifting:" << select << std::endl; select = select.reshift_expressions(minColumnShift_); L << std::endl << "SELECT_LIST after reshifting:" << select << std::endl; if (where) { L << std::endl << "WHERE before reshifting:" << *where << std::endl; where = where->reshift(minColumnShift_); L << std::endl << "WHERE after reshifting:" << *where << std::endl; } if (!order_by.first.empty()) { order_by.first = order_by.first.reshift_expressions(minColumnShift_); } } maxColumnShift_ = 0; minColumnShift_ = 0; if (group_by.size()) { Log::info() << "GROUP BY clause seen and ignored. Non aggregated values on select list " "will be used instead." << std::endl; } SQLOutput* outputEndpoint = nullptr; std::vector> newOutputs; if (!into.empty()) { newOutputs.emplace_back(session_.newFileOutput(into)); outputEndpoint = newOutputs.back().get(); } else { outputEndpoint = &session_.output(); } if (order_by.first.size()) { newOutputs.emplace_back(new SQLOrderOutput(*outputEndpoint, order_by)); outputEndpoint = newOutputs.back().get(); } if (distinct) { newOutputs.emplace_back(new SQLDistinctOutput(*outputEndpoint)); outputEndpoint = newOutputs.back().get(); } r = new SQLSelect(select, fromTables, where, *outputEndpoint, std::move(newOutputs)); maxColumnShift_ = 0; return r; } // MetaData SQLSelectFactory::toODAColumns(SQLSession& session, const TableDef& tableDef) //{ // std::ostream& L(eckit::Log::debug()); // // ColumnDefs columnDefs (tableDef.columns()); // MetaData md(0); // for (size_t i(0); i < columnDefs.size(); ++i) // { // ColumnDef& c (columnDefs[i]); // L << " " << c.name() << ":" << c.type() << std::endl; // // SchemaAnalyzer& a (session.currentDatabase().schemaAnalyzer()); // if (a.isBitfield(c.name())) { // const BitfieldDef& bf ( a.getBitfieldTypeDefinition(c.name()) ); // md.addBitfield(c.name(), bf ); // } // else { // ColumnType type (Column::type(c.type())); // if (type == BITFIELD) // md.addBitfield(c.name(), c.bitfieldDef()); // else // md.addColumn(c.name(), c.type()); // } // // ASSERT( &md[i]->coder() ); // } // return md; //} // SQLOutput& SQLSelectFactory::createOutput (const std::string& into, size_t orderBySize) //{ //// TODO: FIXME ////size_t maxOpenFiles ( ! context ? 100 : atoi(context->environment().lookup("maxOpenFiles", "100", ///*context).c_str())); // size_t maxOpenFiles ( 100); ////TODO: pass parameter into to defaultFormat // SQLOutput *r (NULL); ////if (config_.outputFormat() == "callback") return session.defaultOutput(); // std::string outputFile ((config_.outputFormat() == "odb") ? config_.outputFile() : into); // Log::debug() << "SQLSelectFactory::createOutput: outputFile: '" << outputFile << "'" << std::endl; // if (! outputFile.size()) // TemplateParameters templateParameters; // TemplateParameters::parse(outputFile, templateParameters); // if (templateParameters.size()) // { // NOTIMP; //// r = new SQLODAOutput(new DispatchingWriter(outputFile, orderBySize ? 1 : maxOpenFiles)); // } // else // { // SchemaAnalyzer& a (session.currentDatabase().schemaAnalyzer()); // if (! a.tableKnown(outputFile)) { // NOTIMP; //// r = new SQLODAOutput >(new Writer<>(outputFile)); // } else // { // NOTIMP; // Log::info() << "Table in the INTO clause known (" << outputFile << ")" << std::endl; //// const TableDef* tableDef (&a.findTable(outputFile)); //// r = new SQLODAOutput >(new Writer<>(outputFile), toODAColumns(session, *tableDef)); // } // } // return r; //} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLStatement.cc0000664000175000017500000000113415161702250020321 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLStatement.h" namespace eckit::sql { SQLStatement::SQLStatement() {} SQLStatement::~SQLStatement() {} void SQLStatement::print(std::ostream& s) const {} } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLTableFactory.cc0000664000175000017500000000524515161702250020743 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLTableFactory.h" #include #include "eckit/exception/Exceptions.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SQLTableFactory& SQLTableFactory::instance() { static SQLTableFactory theInstance; return theInstance; } SQLTable* SQLTableFactory::build(SQLDatabase& owner, const std::string& name, const std::string& location) { std::string location2 = location.size() ? location : name; // This sub-scope looks to be entirely superflouse. // See ECKIT-473 // In this case, the cray compiler (8.6, 8.7) are optimising out the call of // std::~lock_guard() in the exception case if no factory succeeds. // This leads to the mutex still being held - at best resulting in an infinite hang // on destruction of global objects at the end of execution. // // The extra scope ensures that the mutex destructor is called _before_ the throw // statement. { std::lock_guard lock(mutex_); for (const auto& factory : factories_) { SQLTable* t = factory->build(owner, name, location2); if (t) { return t; } } } throw UserError("No SQL table could be built for " + name + " (" + location2 + ")", Here()); } void SQLTableFactory::enregister(SQLTableFactoryBase* f) { std::lock_guard lock(mutex_); ASSERT(f); ASSERT(std::find(factories_.begin(), factories_.end(), f) == factories_.end()); factories_.push_back(f); } void SQLTableFactory::deregister(SQLTableFactoryBase* f) { std::lock_guard lock(mutex_); auto it = std::find(factories_.begin(), factories_.end(), f); ASSERT(it != factories_.end()); factories_.erase(it); } //---------------------------------------------------------------------------------------------------------------------- SQLTableFactoryBase::SQLTableFactoryBase() { SQLTableFactory::instance().enregister(this); } SQLTableFactoryBase::~SQLTableFactoryBase() { SQLTableFactory::instance().deregister(this); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/sqll.l0000775000175000017500000001600115161702250016620 0ustar alastairalastair%option reentrant %option extra-type="void *" %option bison-bridge %option yylineno %{ #define YY_USER_ACTION do { \ struct yyguts_t * yyg ((struct yyguts_t*) yyscanner); \ do { (void)(yyg); } while( 0 ); \ /* fprintf(stderr, "location <%d,%d>,<%d,%d>\n", yyg->yylloc_r.first_line, yyg->yylloc_r.first_column, yyg->yylloc_r.last_line, yyg->yylloc_r.last_column); */ \ } while (0); /* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include // http://flex.sourceforge.net/manual/Multiple-Input-Buffers.html#Multiple-Input-Buffers class Stack { static const size_t maxIncludeDepth = 2000; public: void push(const std::string& sql, const std::string& yypath, YY_BUFFER_STATE state, eckit_sql_scan_t scanner) { if (stack_.size() > maxIncludeDepth) throw eckit::UserError("Includes nested too deeply"); stack_.push_back(state); currentBuffer_ = eckit_sql__scan_bytes(sql.c_str(), sql.size(), scanner); eckit_sql__switch_to_buffer(currentBuffer_, scanner); } int pop(eckit_sql_scan_t scanner) { if (stack_.size() <= 1) return 1; YY_BUFFER_STATE currentState (stack_.back()); stack_.pop_back(); eckit_sql__delete_buffer(currentBuffer_, scanner); eckit_sql__switch_to_buffer(currentState, scanner); return 0; } private: YY_BUFFER_STATE currentBuffer_; std::vector stack_; }; typedef Stack include_stack; /* %option noyywrap */ %} IDENT [_A-Za-z]+[_0-9A-Za-z]* VAR [$][_0-9A-Za-z]+ NUMB [-]?[0-9]+(\.[0-9]*)? SEMICOLON [;] %s LEX_ORDERBY %s LEX_CREATE %x incl %% #include BEGIN(incl); @[lL][iI][nN][kK] return LINK; [sS][eE][tT] return SET; [dD][aA][tT][aA][bB][aA][sS][eE] return DATABASE; [sS][eE][lL][eE][cC][tT] return SELECT; [iI][nN][tT][oO] return INTO; [fF][rR][oO][mM] return FROM; [wW][hH][eE][rR][eE] return WHERE; [iI][nN][sS][eE][rR][tT] return INSERT; [vV][aA][lL][uU][eE][sS] return VALUES; [aA][nN][dD] return AND; [oO][rR] return OR; [oO][nN] return ON; [iI][sS] return IS; [nN][uU][lL][lL] return NIL; [nN][oO][tT] return NOT; [cC][oO][uU][nN][tT] return COUNT; [eE][xX][iI][tT] return EXIT; [vV][iI][eE][wW] { BEGIN 0; return VIEW; } [cC][rR][eE][aA][tT][eE] { BEGIN LEX_CREATE; return CREATE; } [sS][cC][hH][eE][mM][aA] { BEGIN 0; return SCHEMA; } [iI][nN][dD][eE][xX] { BEGIN 0; return INDEX; } [tT][aA][bB][lL][eE] { BEGIN 0; return TABLE; } [tT][yY][pP][eE] { BEGIN 0; return TYPE; } [tT][yY][pP][eE][dD][eE][fF] { BEGIN 0; return TYPEDEF; } [tT][yY][pP][eE][oO][fF] return TYPEOF; [bB][eE][tT][wW][eE][eE][nN] return BETWEEN; [dD][iI][sS][tT][iI][nN][cC][tT] return DISTINCT; [aA][lL][lL] return ALL; [mM][iI][sS][sS][iI][nN][gG] return NIL; [gG][rR][oO][uU][pP] return GROUP; [oO][rR][dD][eE][rR] { BEGIN LEX_ORDERBY; return ORDER; } [bB][yY] return BY; [iI][nN] return IN; [rR][eE][aA][dD][oO][nN][lL][yY] return READONLY; [uU][pP][dD][aA][tT][eE][dD] return UPDATED; [nN][oO][rR][eE][oO][rR][dD][eE][rR] return NOREORDER; [sS][aA][fF][eE][gG][uU][aA][rR][dD] return SAFEGUARD; [tT][eE][mM][pP][oO][rR][aA][rR][yY] return TEMPORARY; [aA][sS][cC] return ASC; [dD][eE][sS][cC] return DESC; {SEMICOLON} { BEGIN 0; return ';'; } [aA][sS] return AS; \# return HASH; [cC][oO][nN][sS][tT][rR][aA][iI][nN][tT] return CONSTRAINT; [uU][nN][iI][qQ][uU][eE] return UNIQUE; [pP][rR][iI][mM][aA][rR][yY] return PRIMARY; [fF][oO][rR][eE][iI][gG][nN] return FOREIGN; [kK][eE][yY] return KEY; [rR][eE][fF][eE][rR][eE][nN][cC][eE][sS] return REFERENCES; [dD][eE][fF][aA][uU][lL][tT] return DEFAULT; [iI][nN][hH][eE][rR][iI][tT][sS] { BEGIN 0; return INHERITS; } [mM][aA][tT][cC][hH] return MATCH; [qQ][uU][eE][rR][yY] return QUERY; [lL][iI][kK][eE] return LIKE; [rR][lL][iI][kK][eE] return RLIKE; [aA][lL][iI][gG][nN] return ALIGN; [rR][eE][sS][eE][tT] return RESET; [dD][uU][aA][lL] return DUAL; [oO][nN][eE][lL][oO][oO][pP][eE][rR] return ONELOOPER; [ \t]* /* eat the whitespace */ [^ \t\n;]+ { /* name of the file to be included */ //cerr << "include " << yytext << std::endl; eckit::PathName fileName(eckit::StringTools::unQuote(yytext)); //cerr << "Opening " << fileName << std::endl; std::string text (SQLSession::readIncludeFile(fileName)); Stack* includeStack (static_cast(((struct SQLYacc::eckit_sql_guts_t*) eckit_sql_scanner)->eckit_sql_extra_r)); ASSERT(includeStack); includeStack->push(text, fileName, YY_CURRENT_BUFFER, eckit_sql_scanner); BEGIN(INITIAL); } <> { //cerr << "** EOF **" << std::endl; eckit_sql_terminate(); } \"|\' { int c, end = yytext[0]; yyleng = 0; while ((c = yyinput(yyscanner)) && c != end) { yytext[yyleng++] = (c == '\\') ? yyinput(yyscanner) : c; if (c == '\n') { struct yyguts_t * yyg = (struct yyguts_t*) yyscanner; ++ yyg->yylineno_r; } } yytext[yyleng++] = 0; yylval->val = yytext; return STRING; } \{ {IDENT} { char *p = yytext; while(*p) { *p = tolower(*p); p++; } yylval->val = yytext; //cerr << "****I " << yylval.val << " ****" << std::endl; return IDENT; } {VAR} { char *p = yytext; while(*p) { *p = tolower(*p); p++; } yylval->val = yytext; //cerr << "****V " << yylval.val << " ****" << std::endl; return VAR; } {NUMB} { yylval->num = atof(yytext); //cerr << "****N " << yylval.num << " ****" << std::endl; return DOUBLE; } "--".* ; "//".* ; [ \t]* ; \n { struct yyguts_t * yyg = (struct yyguts_t*) yyscanner; ++ yyg->yylineno_r; } "=" return EQ; "==" return EQ; "!=" return NE; "<>" return NE; ">=" return GE; "<=" return LE; "!" return NOT; . return *yytext; %% eckit-2.0.7/src/eckit/sql/SchemaComponents.h0000664000175000017500000000373115161702250021112 0ustar alastairalastair/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // @author Simon Smart // @date January 2019 #ifndef eckit_sql_SchemaComponents_H #define eckit_sql_SchemaComponents_H #include #include #include "eckit/sql/SQLTypedefs.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class ColumnDef { public: // methods ColumnDef(const std::string& name, const std::string& type, const BitfieldDef& bitfield); // const Range& range, // const std::string& defaultValue); ColumnDef(); ~ColumnDef(); const std::string& name() const { return name_; } const std::string& type() const { return type_; } bool isBitfield() const { return !bitfield_.first.empty(); } const BitfieldDef& bitfield() const { return bitfield_; } private: // members std::string name_; std::string type_; BitfieldDef bitfield_; }; using ColumnDefs = std::vector; //---------------------------------------------------------------------------------------------------------------------- class TableDef { public: // methods TableDef(const std::string& name, const ColumnDefs& columns); ~TableDef(); const std::string& name() const { return name_; } const ColumnDefs& columns() const { return columns_; } private: // members std::string name_; ColumnDefs columns_; }; using TableDefs = std::vector; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLMATCHSubquerySession.h0000664000175000017500000000306615161702250022165 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLMATCHSubquerySession.h // Piotr Kuchta - ECMWF Octover 2015 #ifndef SQLMATCHSubquerySession_H #define SQLMATCHSubquerySession_H #include "eckit/sql/SQLSession.h" namespace eckit { class ReaderIterator; } namespace eckit { class SelectIterator; } namespace eckit { namespace sql { namespace expression { namespace function { class FunctionMATCH; } } // namespace expression } // namespace sql } // namespace eckit namespace eckit { namespace sql { class SQLMATCHSubquerySession : public SQLSession { public: SQLMATCHSubquerySession(expression::function::FunctionMATCH&); ~SQLMATCHSubquerySession(); SQLStatement* statement(); private: // No copy allowed SQLMATCHSubquerySession(const SQLMATCHSubquerySession&); SQLMATCHSubquerySession& operator=(const SQLMATCHSubquerySession&); SQLStatement* statement_; expression::function::FunctionMATCH& f_; // -- Overridden methods void statement(SQLStatement*); SQLOutput* defaultOutput(); // -- Friends // friend std::ostream& operator<<(std::ostream& s,const SQLMATCHSubquerySession& p) // { p.print(s); return s; } }; } // namespace sql } // namespace eckit #endif eckit-2.0.7/src/eckit/sql/SQLSelect.cc0000664000175000017500000006131515161702250017603 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLSelect.h" #include #include "eckit/config/LibEcKit.h" #include "eckit/log/BigNum.h" #include "eckit/log/Log.h" #include "eckit/sql/SQLColumn.h" #include "eckit/sql/SQLDatabase.h" #include "eckit/sql/SQLOutput.h" #include "eckit/sql/SQLTable.h" #include "eckit/sql/expression/ColumnExpression.h" #include "eckit/sql/expression/ConstantExpression.h" #include "eckit/sql/expression/OrderByExpressions.h" #include "eckit/sql/expression/SQLExpressionEvaluated.h" #include "eckit/sql/expression/SQLExpressions.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- using namespace expression; SQLSelect::SQLSelect(const Expressions& columns, const std::vector>& tables, std::shared_ptr where, SQLOutput& output, std::vector>&& ownedOutputs) : select_(columns), where_(where), simplifiedWhere_{nullptr}, ownedOutputs_(std::move(ownedOutputs)), output_(output), aggregatedResultsIterator_(aggregatedResults_.end()), count_(0), total_(0), skips_(0), aggregate_(false), mixedAggregatedAndScalar_(false), doOutputCached_(false) { // TODO: Convert tables_, allTables_ to use references rather than pointers. for (const SQLTable& t : tables) { tables_.push_back(&t); } } SQLSelect::~SQLSelect() {} const SQLTable& SQLSelect::findTable(const std::string& name) const { std::set names; for (const SQLTable* table : tables_) { if (table->hasColumn(name)) { names.insert(table); } } if (names.size() == 0) { throw eckit::UserError("Can't find a table for", name); } if (names.size() != 1) { throw eckit::UserError("Ambiguous column name", name); } Log::debug() << "SQLSelect::findTable: name='" << name << "'" << std::endl; return **names.begin(); } void SQLSelect::ensureFetch(const SQLTable& table, const std::string& columnName) { // Add the column to the fetch list associated with the table const SQLColumn& column(table.column(columnName)); std::string fullname = column.fullName(); allTables_.insert(&table); if (tablesToFetch_.find(&table) == tablesToFetch_.end()) { tablesToFetch_[&table] = SelectOneTable(&table); } auto& fetch(tablesToFetch_[&table].fetch_); if (std::find_if(fetch.begin(), fetch.end(), [&](const SQLColumn& c) { return &c == &column; }) == fetch.end()) { fetch.push_back(column); // tablesToFetch_[&table].fetchSizeDoubles_.push_back(column.dataSizeDoubles()); // This will create the value if it doesn't exist ValueLookup& value(values_[fullname]); tablesToFetch_[&table].values_.push_back(&value); } } SQLSelect::ValueLookup& SQLSelect::column(const std::string& name, const SQLTable* table) { // if this ASSERT fails, we are likely missing a call to preprepare() on the expression // in SQlSelect::prepareExecute() ASSERT(table); const SQLColumn& column(table->column(name)); std::string fullname = column.fullName(); Log::debug() << "Accessing column " << fullname << std::endl; auto it = values_.find(fullname); ASSERT(it != values_.end()); return it->second; } const type::SQLType* SQLSelect::typeOf(const std::string& name, const SQLTable* table) const { if (!table) { table = &findTable(name); } const SQLColumn& column(table->column(name)); const type::SQLType& type = column.type(); return type.subType(name); // This should take care of bitfields } static bool compareTables(SelectOneTable* a, SelectOneTable* b) { // #if 1 // if(&(a->table_->owner()) != &(b->table_->owner())) return a->table_->owner().name() < b->table_->owner().name(); // #else // return a->table_->index() < b->table_->index(); // #endif } inline bool SQLSelect::resultsOut() { return output_.output(select_); } std::shared_ptr SQLSelect::findAliasedExpression(const std::string& alias) { for (size_t i(0); i < select_.size(); ++i) { if (select_[i]->title() == alias) { return select_[i]; } } return nullptr; } void SQLSelect::prepareExecute() { reset(); // Associate ColumnExpressions, SQLTable and SQLColumns. // n.b. it is a bit yucky to do the prepare() in two steps, but this allows us to // allocate buffers and associate them with iterators appropriately. for (Expressions::iterator c = select_.begin(); c != select_.end(); ++c) { (*c)->preprepare(*this); } if (where_) { where_->preprepare(*this); } output_.preprepare(*this); // If no tables have been required, but there are expressions, it implies that we are // just using functions that don't depend on the data (e.g. rownumber()). Just ensure // that we do something... if (!select_.empty() && tablesToFetch_.empty()) { for (const SQLTable* table : tables_) { tablesToFetch_[table] = SelectOneTable(table); } } // Construct the cursors that will be used for the selection, and pass these in to the // constructed cursors/iterators for (auto& tablePair : tablesToFetch_) { SelectOneTable& tbl(tablePair.second); // n.b. tablePair.first is only const to enable other functions to be const. But // it belongs to this structure, and we are a non-const fn, so this is ok. SQLTable* sqlTable = const_cast(tablePair.first); cursors_.emplace_back(tbl.table_->iterator( tbl.fetch_, [this, sqlTable](SQLTableIterator& cursor) { refreshCursorMetadata(sqlTable, cursor); })); cursors_.back()->rewind(); refreshCursorMetadata(sqlTable, *cursors_.back()); } // Prepare the columns for (Expressions::iterator c = select_.begin(); c != select_.end(); ++c) { if ((*c)->isAggregate()) { aggregated_.push_back(*c); mixedResultColumnIsAggregated_.push_back(true); } else { nonAggregated_.push_back(*c); mixedResultColumnIsAggregated_.push_back(false); } (*c)->prepare(*this); Log::debug() << "SQLSelect::prepareExecute: '" << *(*c) << "'" << std::endl; } ASSERT(select_.size() == mixedResultColumnIsAggregated_.size()); ASSERT(select_.size() == aggregated_.size() + nonAggregated_.size()); output_.prepare(*this); if (aggregated_.size()) { aggregate_ = true; Log::debug() << "SELECT is aggregated" << std::endl; if (aggregated_.size() != select_.size()) { mixedAggregatedAndScalar_ = true; Log::debug() << "SELECT has aggregated and non-aggregated results" << std::endl; } } std::shared_ptr where(where_); if (where) { where->prepare(*this); bool more = true; while (more) { more = false; std::shared_ptr w(where->simplify(more)); if (w) { where = w; } simplifiedWhere_ = where; } Log::debug() << "Simplified WHERE " << *where << std::endl; if (where->isConstant()) { bool missing = false; if (where->eval(missing)) { Log::debug() << "WHERE condition always true" << std::endl; where = nullptr; } else { Log::debug() << "WHERE condition always false" << std::endl; return; } } } // Check for links for (std::set::iterator j = allTables_.begin(); j != allTables_.end(); ++j) { const SQLTable* table1 = *j; const std::string& name1 = table1->name(); for (std::set::iterator k = allTables_.begin(); k != allTables_.end(); ++k) { const SQLTable* table2 = *k; SelectOneTable& x = tablesToFetch_[table2]; if (table1->hasLinkTo(*table2)) { const std::string& name2 = table2->name(); // That can happen for 'aligned' tables if (x.column_) { ASSERT(table2 == x.table2_); Log::warning() << "Ignoring link " << name1 << "->" << name2 << ", using " << x.table1_->fullName() << "->" << x.table2_->fullName() << std::endl; continue; } Log::debug() << "Using link " << table1->fullName() << "->" << table2->fullName() << std::endl; // std::string o = name2 + ".offset"; ValueLookup offset = column(o, table1); std::string l = name2 + ".length"; ValueLookup length = column(l, table1); // There should not be 2 tables with a link on the same table // TODO: Test and fixme // ASSERT(x.offset_ == 0); // ASSERT(x.length_ == 0); ASSERT(x.offset_.first == 0); ASSERT(x.length_.second == 0); ASSERT(x.column_ == nullptr); x.offset_ = offset; x.length_ = length; x.column_ = &table1->column(name2 + ".offset"); x.table1_ = table1; x.table2_ = table2; } } } // ------------------------------------ for (TableMap::iterator j = tablesToFetch_.begin(); j != tablesToFetch_.end(); ++j) { sortedTables_.push_back(&(*j).second); } if (where) { // Try to analyse where expression::Expressions e; if (!where->andSplit(e)) { e.push_back(where); } for (size_t i = 0; i < e.size(); ++i) { Log::debug() << "WHERE AND split " << *(e[i]) << std::endl; // Get tables accessed std::set t; e[i]->tables(t); for (std::set::iterator j = t.begin(); j != t.end(); ++j) { Log::debug() << " tables -> " << (*j)->fullName() << std::endl; } if (t.size() == 1) { const SQLTable* table = *(t.begin()); tablesToFetch_[table].check_.push_back(e[i]); Log::debug() << "WHERE quick check for " << table->fullName() << " " << (*e[i]) << std::endl; } } } // Needed, for example, if we do: select count(*) from "file.oda" if (sortedTables_.size() == 0) { for (std::vector::iterator i = tables_.begin(); i != tables_.end(); ++i) { sortedTables_.push_back(new SelectOneTable(*i)); // TODO: release the objects! } } std::sort(sortedTables_.begin(), sortedTables_.end(), compareTables); Log::debug() << "TABLE order " << std::endl; for (SortedTables::iterator k = sortedTables_.begin(); k != sortedTables_.end(); ++k) { Log::debug() << (*k)->table_->fullName() << " " << (*k)->order_ << std::endl; for (size_t i = 0; i < (*k)->check_.size(); i++) { Log::debug() << " QUICK CHECK " << *((*k)->check_[i]) << std::endl; } for (size_t i = 0; i < (*k)->index_.size(); i++) { Log::debug() << " INDEX CHECK " << *((*k)->index_[i]) << std::endl; } } // Add the multi-table quick checks if (where) { expression::Expressions e; if (!where->andSplit(e)) { e.push_back(where); } std::set ordered; for (SortedTables::iterator k = sortedTables_.begin(); k != sortedTables_.end(); ++k) { const SQLTable* table = (*k)->table_; ordered.insert(table); for (size_t i = 0; i < e.size(); ++i) { if (e[i]) { // Get tables accessed std::set t; e[i]->tables(t); if (t.size() != 1) { bool ok = true; for (std::set::iterator j = t.begin(); j != t.end(); ++j) { if (ordered.find(*j) == ordered.end()) { ok = false; } } if (ok) { (*k)->check_.push_back(e[i]); Log::debug() << "WHERE multi-table quick check for " << table->fullName() << " " << (*e[i]) << std::endl; e[i] = nullptr; } } else { e[i] = nullptr; } } } } // Add what's left to last table for (size_t i = 0; i < e.size(); ++i) { if (e[i]) { sortedTables_.back()->check_.push_back(e[i]); } } where = nullptr; } // Debug output Log::debug() << "SQLSelect:prepareExecute: TABLE order:" << std::endl; for (SortedTables::iterator k = sortedTables_.begin(); k != sortedTables_.end(); ++k) { Log::debug() << "SQLSelect:prepareExecute: TABLE order " << (*k)->table_->fullName() << " " << (*k)->order_ << std::endl; for (size_t i = 0; i < (*k)->check_.size(); i++) { Log::debug() << " QUICK CHECK " << *((*k)->check_[i]) << std::endl; } } } unsigned long long SQLSelect::execute() { prepareExecute(); process(); postExecute(); return count_; } void SQLSelect::postExecute() { output_.flush(); output_.cleanup(*this); if (simplifiedWhere_) { simplifiedWhere_->cleanup(*this); } for (expression::Expressions::iterator c(select_.begin()); c != select_.end(); ++c) { (*c)->cleanup(*this); } Log::debug() << "Matching row(s): " << BigNum(output_.count()) << " out of " << BigNum(total_) << std::endl; Log::debug() << "Skips: " << BigNum(skips_) << std::endl; reset(); } void SQLSelect::refreshCursorMetadata(SQLTable* table, SQLTableIterator& cursor) { auto it = tablesToFetch_.find(table); ASSERT(it != tablesToFetch_.end()); SelectOneTable& tbl(it->second); const double* data(cursor.data()); const std::vector offsets(cursor.columnOffsets()); const std::vector doublesSizes(cursor.doublesDataSizes()); const std::vector hasMissing(cursor.columnsHaveMissing()); const std::vector missingValues(cursor.missingValues()); for (size_t i = 0; i < tbl.fetch_.size(); i++) { std::string fullname(tbl.fetch_[i].get().fullName()); // ASSERT is no longer true if using to refresh values // ASSERT(values_.find(fullname) == values_.end()); // HACK ALERT // Our output functionality gets the width of strings from the column type. // but the width can change on refreshCursorMetadata // --> We need to update the column type. // This is a function of the table. Aaaarg. Makes this horribly not parallelisable. // eckit::sql needs a rewrite. Will enable us to work around this shit. table->updateColumnDoublesWidth(fullname, doublesSizes[i]); // Aaaargh, we need to know when we are looking at missing values. But these can change // down the column. As can 'hasMissing' table->updateColumnMissingValues(fullname, hasMissing[i], missingValues[i]); // This will create the value if it does not exist. std::pair& value(values_[fullname]); value.first = &data[offsets[i]]; } for (Expressions::iterator c = select_.begin(); c != select_.end(); ++c) { (*c)->updateType(*this); } if (where_) { where_->prepare(*this); } output_.updateTypes(*this); } void SQLSelect::reset() { aggregate_ = false; mixedAggregatedAndScalar_ = false; doOutputCached_ = false; aggregated_.clear(); nonAggregated_.clear(); aggregatedResults_.clear(); mixedResultColumnIsAggregated_.clear(); values_.clear(); tablesToFetch_.clear(); allTables_.clear(); sortedTables_.clear(); skips_ = total_ = 0; output_.reset(); cursors_.clear(); count_ = 0; } bool SQLSelect::writeOutput() { std::shared_ptr& where(simplifiedWhere_); // if (where) Log::info() << "SQLSelect::output: where: " << *where << std::endl; bool newRow = false; bool missing = false; double value; if (!where || (((value = where->eval(missing)) || !value) // !value for the 'WHERE 0' case, ODB-106 && !missing)) { if (!aggregate_) { newRow = resultsOut(); } else { size_t n = select_.size(); if (!mixedAggregatedAndScalar_) { for (size_t i = 0; i < n; i++) { select_[i]->partialResult(); } } else { // For each set of non-aggregated values, keep track of the aggregated values // n.b. newRow=false, as we are accumulating the values OrderByExpressions nonAggregatedValues; for (size_t i = 0; i < nonAggregated_.size(); ++i) { nonAggregatedValues.emplace_back(std::make_shared(*nonAggregated_[i])); } AggregatedResults::iterator results = aggregatedResults_.find(nonAggregatedValues); if (results == aggregatedResults_.end()) { Expressions& aggregated = aggregatedResults_[nonAggregatedValues]; for (const auto& expr : aggregated_) { aggregated.emplace_back(expr->clone()); } } Expressions& aggregated = aggregatedResults_[nonAggregatedValues]; for (size_t i = 0; i < aggregated.size(); ++i) { aggregated[i]->partialResult(); } } } } return newRow; } unsigned long long SQLSelect::process() { ASSERT(cursors_.size() != 0); ASSERT(count_ == 0); while (processOneRow()) { /* Intentionally blank */; } return count_; } bool SQLSelect::processNextTableRow(size_t tableIndex) { ASSERT(cursors_.size() > tableIndex); ASSERT(cursors_.size() == sortedTables_.size()); /// For one table, obtain the next row that also validates, or return false if there is not one. SelectOneTable& fetchTable(*sortedTables_[tableIndex]); total_++; while (cursors_[tableIndex]->next()) { // Extract the missing values for (size_t i = 0; i < fetchTable.fetch_.size(); i++) { fetchTable.values_[i]->second = fetchTable.fetch_[i].get().isMissingValue(fetchTable.values_[i]->first); } // Test thereturned row against the validation conditions. bool ok = true; for (auto& check : fetchTable.check_) { bool missing = false; if (!check->eval(missing) || missing) { ok = false; break; } } if (ok) { return true; } skips_++; total_++; } // If no row was found, then ensure total_ was not incremented. total_--; return false; } bool SQLSelect::processOneRow() { // n.b. it is acceptable for fromTables.size() == 0, if the expressions // are all constant. So we just check the number of used tables. ASSERT(cursors_.size() == sortedTables_.size()); // If we are using and output iterator that caches the output, and are now replaying // that data (say using OrderBy), then do that. if (doOutputCached_) { if (output_.cachedNext()) { count_++; return true; } doOutputCached_ = false; return false; } // If this is the first retrieve, we need to initialise all tables if (count_ == 0) { for (size_t idx = 0; idx < cursors_.size(); idx++) { if (!processNextTableRow(idx)) { return false; // If false, there is no data } } if (writeOutput()) { count_++; return true; ; } } // Otherwise, start by incrementing the first table. If that is exhausted, reset that table // and increment the second, and continue until we have enumerated all possible combinations // of valid data across the tables. if (!mixedAggregatedAndScalar_ || aggregatedResultsIterator_ == aggregatedResults_.end()) { for (size_t idx = 0; idx < cursors_.size(); idx++) { // n.b. keep going until writeOutput() has done something - i.e. a row has been // returned. This allows us to have filtering/unique/aggregation in the Output while (processNextTableRow(idx)) { if (writeOutput()) { count_++; return true; } } // If we have exhausted the available rows, rewind and increment those in the next table if (idx != cursors_.size() - 1) { cursors_[idx]->rewind(); ASSERT(processNextTableRow(idx)); } } } // If we are considering mixed aggregate/non-aggregate results, then we need to output // them here // We put this here rather than in postExecute such that the Select class in odb can // iterate over one entry at a time. if (aggregatedResultsIterator_ == aggregatedResults_.end()) { aggregatedResultsIterator_ = aggregatedResults_.begin(); } else { ++aggregatedResultsIterator_; } while (aggregatedResultsIterator_ != aggregatedResults_.end()) { const OrderByExpressions& nonAggregated = aggregatedResultsIterator_->first; Expressions& aggregated = aggregatedResultsIterator_->second; Expressions results; size_t ai = 0; size_t ni = 0; for (size_t i = 0; i < mixedResultColumnIsAggregated_.size(); i++) { if (mixedResultColumnIsAggregated_[i]) { results.push_back(aggregated[ai++]); } else { results.push_back(nonAggregated[ni++]); } } if (output_.output(results)) { count_++; return true; } ++aggregatedResultsIterator_; } // If this is an aggregate (not mixed aggregate) case, then we are done the // first time we pass through. But ensure that the caller is told that there // is at least some data! if (aggregate_ && !mixedAggregatedAndScalar_ && count_ == 0) { resultsOut(); count_++; return true; } // If we still get here, we might be using an output that caches all the results // (e.g. OrderBy). Give it the chance to do its output if (output_.cachedNext()) { doOutputCached_ = true; count_++; return true; } // Nothing to return. return false; } void SQLSelect::print(std::ostream& s) const { s << "SELECT"; char sep = ' '; for (expression::Expressions::const_iterator c = select_.begin(); c != select_.end(); ++c) { s << sep << *(*c); sep = ','; } s << " FROM"; sep = ' '; for (std::vector::const_iterator t = tables_.begin(); t != tables_.end(); ++t) { s << sep << (*t)->name(); sep = ','; } if (where_.get()) { s << " WHERE " << *where_; } s << " " << output_; } expression::Expressions SQLSelect::output() const { return select_; } std::vector SQLSelect::outputFiles() const { return outputFiles_; } void SQLSelect::outputFiles(const std::vector& files) { outputFiles_ = files; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SchemaComponents.cc0000664000175000017500000000227415161702250021251 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SchemaComponents.h" using namespace eckit; namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- ColumnDef::ColumnDef(const std::string& name, const std::string& type, const BitfieldDef& bitfield) : // const Range &range, // const std::string& defaultValue) {} name_(name), type_(type), bitfield_(bitfield) {} ColumnDef::ColumnDef() {} ColumnDef::~ColumnDef() {} TableDef::TableDef(const std::string& name, const ColumnDefs& columns) : name_(name), columns_(columns) {} TableDef::~TableDef() {} //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLMATCHSubquerySessionOutput.cc0000664000175000017500000000370015161702250023537 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLMATCHSubquerySessionOutput.h" #include "eckit/exception/Exceptions.h" #include "odb_api/Expressions.h" #include "odb_api/FunctionMATCH.h" using namespace std; namespace eckit { namespace sql { SQLMATCHSubquerySessionOutput::SQLMATCHSubquerySessionOutput(const SQLMATCHSubquerySessionOutput& other) : f_(other.f_), count_(0) {} SQLMATCHSubquerySessionOutput::SQLMATCHSubquerySessionOutput(odb::sql::expression::function::FunctionMATCH& f) : f_(f), count_(0) {} SQLMATCHSubquerySessionOutput& SQLMATCHSubquerySessionOutput::operator=(const SQLMATCHSubquerySessionOutput& other) { f_ = other.f_; count_ = other.count_; return *this; } SQLMATCHSubquerySessionOutput::~SQLMATCHSubquerySessionOutput() {} void SQLMATCHSubquerySessionOutput::print(std::ostream& s) const { s << "SQLMATCHSubquerySessionOutput"; } void SQLMATCHSubquerySessionOutput::size(int) {} void SQLMATCHSubquerySessionOutput::reset() {} void SQLMATCHSubquerySessionOutput::flush() {} bool SQLMATCHSubquerySessionOutput::output(const expression::Expressions& results) { const size_t nCols(results.size()); vector v(nCols); for (size_t i(0); i < nCols; ++i) { bool missing(false); v[i] = results[i]->eval(missing); } f_.collect(v); ++count_; return true; } unsigned long long SQLMATCHSubquerySessionOutput::count() { return count_; } void SQLMATCHSubquerySessionOutput::prepare(SQLSelect& sql) {} void SQLMATCHSubquerySessionOutput::cleanup(SQLSelect& sql) {} } // namespace sql } // namespace eckit eckit-2.0.7/src/eckit/sql/sqly.y0000775000175000017500000005625715161702250016673 0ustar alastairalastair%pure-parser %lex-param {void * scanner} %lex-param {eckit::sql::SQLSession* session} %parse-param {void * scanner} %parse-param {eckit::sql::SQLSession* session} %{ /* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #define YYMAXDEPTH 500 using namespace SQLYacc; typedef SQLExpression* SQLExpressionPtr; // For casts. // n.b. Because all elements need to be valid, we cannot use references or reference_wrappers // --> we use raw pointers to items. // --> Convert to references on the output boundary struct YYSTYPE { std::shared_ptr exp; SQLTable* table; double num; std::string val; std::vector list; Expressions explist; std::pair,bool> orderexp; std::pair> orderlist; std::vector> tablist; ColumnDef coldef; std::vector coldefs; std::pair bfdef; std::vector> bfdefs; bool bol; std::pair range; }; #ifdef YY_DECL #undef YY_DECL #endif #define YY_DECL int eckit_sql_lex (YYSTYPE *eckit_sql_lval_param, eckit_sql_scan_t eckit_sql_scanner, eckit::sql::SQLSession*) #define YYSTYPE_IS_DECLARED 1 YY_DECL; Expressions emptyExpressionList; %} %token STRING %token IDENT %token VAR %token DOUBLE %token SEMICOLON %token LINK %token TYPEOF %token READONLY %token UPDATED %token NOREORDER %token SAFEGUARD %token EXIT %token SELECT %token INTO %token FROM %token SET %token DATABASE %token COUNT %token WHERE %token GROUP %token ORDER %token BY %token INSERT %token VALUES %token CREATE %token SCHEMA %token VIEW %token INDEX %token TABLE %token TYPE %token TYPEDEF %token TEMPORARY %token INHERITS %token DEFAULT %token CONSTRAINT %token UNIQUE %token PRIMARY %token FOREIGN %token KEY %token REFERENCES %token EQ %token GE %token LE %token NE %token IN %token NOT %token AND %token OR %token ON %token IS %token AS %token NIL %token ALL %token DISTINCT %token BETWEEN %token ASC %token DESC %token HASH %token LIKE %token RLIKE %token MATCH %token QUERY %token ALIGN %token RESET %token DUAL %token ONELOOPER %type expression assignment_rhs; %type column factor term conjonction disjonction condition atom_or_number vector_index optional_hash; %type where; %type expression_list group_by; %type table_list; %type
table %type table_reference; %type from; %type order_by order_list; %type order; %type select_list select_list_; %type select select_; %type into; %type func relational_operator; %type data_type column_name table_name; %type column_def; %type column_def_list column_def_list_; %type bitfield_def %type bitfield_def_list bitfield_def_list_ %type distinct; %type bitfield_ref default_value; %type vector_range_decl; %% start : statements { session->currentDatabase().setLinks(); } ; statements : statement ';' | statements statement ';' ; statement: select_statement // | create_view_statement { session->statement($1); } // | insert_statement { session->statement(session->insertFactory().create(*session, /*InsertAST* */ ($1))); } | set_statement // | create_schema_statement // | create_index_statement | create_type_statement | create_table_statement // | readonly_statement // | updated_statement // | noreorder_statement // | safeguard_statement // | exit_statement // | align_statement // | reset_statement // | onelooper_statement // | error | empty ; // //exit_statement: EXIT { return 0; } // ; // //readonly_statement: READONLY // ; // //updated_statement: UPDATED // ; // //noreorder_statement: NOREORDER // ; // //safeguard_statement: SAFEGUARD // ; // //align_statement: ALIGN '(' IDENT ',' IDENT ')' ; //onelooper_statement: ONELOOPER '(' IDENT ',' IDENT ')' ; //reset_statement: RESET ALIGN ; //reset_statement: RESET ONELOOPER ; // //create_schema_statement: CREATE SCHEMA schema_name schema_element_list // { // session->currentDatabase().schemaAnalyzer().endSchema(); // } // ; // //schema_name: IDENT // { // session->currentDatabase().schemaAnalyzer().beginSchema($1); // } // ; // //schema_element_list: schema_element // | schema_element_list schema_element // ; // //schema_element: create_table_statement // | create_view_statement // | empty // ; // //create_index_statement: CREATE INDEX IDENT TABLE IDENT // { // session->createIndex($3,$5); // } // | CREATE INDEX IDENT '@' IDENT // { // session->createIndex($3,$5); // } // ; create_type_statement: create_type IDENT as_or_eq '(' bitfield_def_list ')' { std::string typeName ($2); std::vector> bfDefs($5); FieldNames fields; Sizes sizes; for (const auto& def : bfDefs) { const std::string& name(def.first); const int& size(def.second); fields.push_back(name); sizes.push_back(size); ASSERT(sizes.back() > 0); } // std::string typeSignature (type::SQLBitfield::make("Bitfield", fields, sizes, typeName.c_str())); session->currentDatabase() .schemaAnalyzer().addBitfieldType(typeName, fields, sizes); //, typeSignature); //cout << "CREATE TYPE " << typeName << " AS " << typeSignature << ";" << std::endl; } ; create_type_statement: create_type IDENT as_or_eq IDENT { type::SQLType::createAlias($4, $2); } ; create_type: CREATE TYPE | TYPEDEF ; as_or_eq: AS | EQ ; bitfield_def_list: bitfield_def_list_ { $$ = $1; } | bitfield_def_list_ ',' { $$ = $1; } ; bitfield_def_list_: bitfield_def { $$ = {}; $$.push_back($1); } | bitfield_def_list_ ',' bitfield_def { $$ = $1; $$.push_back($3); } ; bitfield_def: column_name data_type { std::string name($1); std::string sz($2); if ((sz.size() != 4 && sz.size() != 5) || sz.substr(0, 3) != "bit") { std::ostringstream ss; ss << "Unexpected bitfield definition: " << name << " --> " << sz; throw eckit::UserError(ss.str(), Here()); } int size = ::strtod(sz.c_str()+3, nullptr); ASSERT(size > 0); $$ = std::make_pair(name, size); } ; //temporary: TEMPORARY { $$ = true; } // | empty { $$ = false; } // ; // //inheritance_list: inheritance_list_ { $$ = $1; } // | inheritance_list_ ',' { $$ = $1; } // ; // //inheritance_list_: IDENT { $$ = std::vector(); $$.insert($$.begin(), $1); } // | inheritance_list_ ',' IDENT { $$ = $1; $$.push_back($3); } // //inherits: INHERITS '(' inheritance_list ')' { $$ = $3; } // | empty { $$ = std::vector(); } // ; // //constraint_list: constraint_list_ { $$ = $1; } // | constraint_list_ ',' { $$ = $1; } // ; // //constraint_list_: constraint { $$ = ConstraintDefs(1, $1); } // | constraint_list_ ',' constraint { $$ = $1; $$.push_back($3); } // | empty { $$ = ConstraintDefs(0); } // ; // //constraint: primary_key { $$ = $1; } // | foreign_key { $$ = $1; } // ; // //primary_key: constraint_name UNIQUE '(' column_reference_list ')' { $$ = ConstraintDef($1, $4); } // | constraint_name PRIMARY KEY '(' column_reference_list ')' { $$ = ConstraintDef($1, $5); } // ; // //foreign_key: constraint_name FOREIGN KEY '(' column_reference_list ')' REFERENCES IDENT '(' column_reference_list ')' // { // $$ = ConstraintDef($1, $5, $8, $10); // } // ; // //constraint_name: CONSTRAINT IDENT { $$ = $2; } // | empty { $$ = std::string(); } // ; // //column_reference_list: column_reference { $$ = std::vector(1, $1); } // | column_reference_list ',' column_reference { $$ = $1; $$.push_back($3); } // ; // //column_reference: IDENT table_reference { $$ = $1 + $2; } // ; // //location: ON STRING { $$ = $2; } // | empty { $$ = ""; } // ; // table_md: AS '(' column_def_list ')' { $$ = $3; } //constraint_list ')' { $$ = make_pair($3, $4); } // | empty { $$ = ColumnDefs(); } //make_pair(ColumnDefs(), ConstraintDefs()); } // ; //create_table_statement: CREATE temporary TABLE table_name table_md inherits location create_table_statement: CREATE TABLE table_name AS '(' column_def_list ')' { std::string name = $3; ColumnDefs cols = $6; TableDef tableDef(name, cols); session->currentDatabase().schemaAnalyzer().addTable(tableDef); } ; // TODO: Do we need ot be able to construct a table name from an expression? table_name: IDENT { $$ = $1; } | IDENT '.' IDENT { $$ = $1 + std::string(".") + $3; } | expression { std::shared_ptr e($1); $$ = e->title(); } ; column_def_list: column_def_list_ { $$ = $1; } | column_def_list_ ',' { $$ = $1; } | empty { $$ = ColumnDefs(); } ; column_def_list_: column_def { $$ = ColumnDefs(); $$.push_back($1); } | column_def_list_ ',' column_def { $$ = $1; $$.push_back($3); } ; column_def: column_name vector_range_decl data_type default_value { // n.b. Ignore vector range decl and default value. These are just for valid schema parsing std::string column_name($1); std::string data_type($3); const BitfieldDef& bitfield(session->currentDatabase().schemaAnalyzer().getBitfieldType(data_type)); $$ = ColumnDef(column_name, data_type, bitfield); } ; vector_range_decl: '[' DOUBLE ']' { $$ = std::make_pair(1, $2); } | '[' DOUBLE ':' DOUBLE ']' { $$ = std::make_pair($2, $4); } | empty { $$ = std::make_pair(0, 0); } ; column_name: IDENT { $$ = $1; } ; data_type: IDENT { $$ = $1; } | LINK { $$ = "@LINK"; } // | TYPEOF '(' column ')' { // std::stringstream ss; // ss << *($3); // std::string columnName (ss.str()); // std::string type (session->currentDatabase().schemaAnalyzer().findColumnType(columnName)); // Log::debug() << "TYPEOF(" << *($3) << ") => " << type << std::endl; // $$ = type; // } ; default_value: DEFAULT expression { std::shared_ptr e($2); $$ = e->title(); } | empty { $$ = std::string(); } ; //create_view_statement: CREATE VIEW IDENT AS select_statement { $$ = $5; } // ; select_statement: SELECT distinct select_list into from where group_by order_by { bool distinct($2); Expressions select_list($3); std::string into($4); std::vector> from ($5); std::shared_ptr where($6); Expressions group_by($7); std::pair> order_by($8); session->setStatement( session->selectFactory().create(distinct, select_list, into, from, where, group_by, order_by) ); } ; distinct: DISTINCT { $$ = true; } | empty { $$ = false; } ; into: INTO IDENT { $$ = $2; } | INTO STRING { $$ = $2; } | empty { $$ = ""; } ; from : FROM table_list { $$ = $2; } | empty { $$ = std::vector>(); } ; where : WHERE expression { $$ = $2; } | { $$ = 0; } ; //insert_statement: INSERT INTO table optional_columns VALUES values // { $$ = InsertAST($3, $4, $6); } // ; // //optional_columns: '(' columns ')' { $$ = $2; } // | empty { $$ = std::vector(); } // ; // //columns: IDENT { $$ = std::vector(); $$.insert($$.begin(), $1); } // | columns ',' IDENT { $$ = $1; $$.push_back($3); } // ; // //values: '(' values_list ')' { $$ = $2; } // ; // //values_list: '?' { $$ = std::vector(); $$.insert($$.begin(), "?" /*$1*/); } // | values_list ',' '?' { $$ = $1; $$.push_back("?"/*$3*/); } // ; assignment_rhs : expression ; //set_statement : SET DATABASE STRING { session->openDatabase($3); } // ; // //set_statement : SET DATABASE STRING AS IDENT { session->openDatabase($3,$5); } // ; set_statement : SET VAR EQ assignment_rhs { //using namespace std; //cout << "== set variable " << $2 << " to "; if ($4) cout << *($4) << std::endl; else cout << "NULL" << std::endl; session->currentDatabase().setVariable($2, $4); } ; bitfield_ref: '.' IDENT { $$ = $2; } | { $$ = std::string(); } ; column: IDENT vector_index table_reference optional_hash { std::string columnName = $1; std::shared_ptr vectorIndex = $2; std::string tableReference($3); // TODO: table_reference should handle . suffix std::shared_ptr pshift($4); std::string bitfieldName; $$ = session->selectFactory().createColumn(columnName, bitfieldName, vectorIndex, tableReference /*TODO: handle . */, pshift); } | IDENT bitfield_ref table_reference optional_hash { std::string columnName = $1; std::string bitfieldName = $2; std::shared_ptr vectorIndex; std::string tableReference($3); // TODO: table_reference should handle . suffix std::shared_ptr pshift ($4); $$ = session->selectFactory().createColumn(columnName, bitfieldName, vectorIndex, tableReference /*TODO: handle . */, pshift); } ; vector_index : '[' expression ']' { $$ = $2; } | empty { $$ = NULL; } ; table_reference: '@' IDENT { $$ = std::string("@") + $2; } | empty { $$ = std::string(""); } ; // n.b. convert SQLTable& into SQLTable* as we need to store a ptr to keep YACC happy // table : IDENT '.' IDENT { $$ = &session->findTable($1, $3); // Multiple databases not supported table : DUAL { $$ = 0; } | IDENT { $$ = &session->findTable($1); } | STRING { $$ = &session->findTable($1); } ; table_list : table { $$ = std::vector>(); if($1) { $$.push_back(*$1); } } | table_list ',' table { $$ = $1; if ($3) { $$.push_back(*$3); } } ; ///*================= SELECT =========================================*/ select_list: select_list_ { $$ = $1; } | select_list_ ',' { $$ = $1; } ; select_list_ : select { $$ = Expressions(); $$.push_back($1); } | select_list_ ',' select { $$ = $1; $$.push_back($3); } ; select: select_ access_decl { $$ = $1; } ; select_: '*' table_reference { $$ = std::make_shared("*", $2); } | IDENT '.' '*' table_reference { $$ = std::make_shared($1, "*", $4); } // | IDENT '[' expression ':' expression ']' table // { // // TODO: Add simillar rule for BitColumnExpression. // bool missing (false); // int begin ($3->eval(missing)); //ASSERT(!missing); // int end ($5->eval(missing)); //ASSERT(!missing); // Table table ($7); // $$ = std::make_shared($1, table.name /*TODO: handle . */, begin, end); // } | expression AS IDENT table_reference { $$ = $1; $$->title($3 + $4); } | expression ; access_decl: UPDATED | READONLY | empty ; ///*================= GROUP BY ======================================*/ group_by: GROUP BY expression_list { $$ = $3; } | empty { $$ = Expressions(); } ; ///*================= ORDER =========================================*/ order_by: ORDER BY order_list { $$ = $3; } | empty { $$ = std::make_pair(Expressions(),std::vector()); } ; order_list : order { $$ = std::make_pair(Expressions(), std::vector()); $$.first.push_back($1.first); $$.second.push_back($1.second); } | order_list ',' order { $$ = $1; $$.first.push_back($3.first); $$.second.push_back($3.second); } ; order : expression DESC { $$ = std::make_pair($1, false); } | expression ASC { $$ = std::make_pair($1, true); } | expression { $$ = std::make_pair($1, true); } ; /*================= EXPRESSION =========================================*/ expression_list : expression { $$ = Expressions(); $$.push_back($1); } | expression_list ',' expression { $$ = $1; $$.push_back($3); } ; optional_hash : HASH DOUBLE { $$ = std::make_shared($2); } | { $$ = std::make_shared(0); } ; atom_or_number : '(' expression ')' { $$ = $2; } | '-' expression { $$ = FunctionFactory::instance().build("-",$2); } | DOUBLE { $$ = std::make_shared($1); } | column | VAR { $$ = session->currentDatabase().getVariable($1); } | '?' DOUBLE { $$ = std::make_shared($2); } | func '(' expression_list ')' { $$ = FunctionFactory::instance().build($1, $3); } | func '(' empty ')' { $$ = FunctionFactory::instance().build($1, emptyExpressionList); } | func '(' '*' ')' { if (std::string("count") != $1) throw eckit::UserError(std::string("Only function COUNT can accept '*' as parameter (") + $1 + ")"); $$ = FunctionFactory::instance().build("count", std::make_shared(1.0)); } | STRING { $$ = std::make_shared($1); } ; func : IDENT { $$ = $1; } | COUNT { $$ = "count"; } ; /* note: a^b^c -> a^(b^c) as in fortran */ factor : factor '*' atom_or_number { $$ = FunctionFactory::instance().build("*",$1,$3); } | factor '/' atom_or_number { $$ = FunctionFactory::instance().build("/",$1,$3); } /* | factor '%' atom_or_number { $$ = std::make_shared($1,$3); } */ | atom_or_number ; term : term '+' factor { $$ = FunctionFactory::instance().build("+",$1,$3); } | term '-' factor { $$ = FunctionFactory::instance().build("-",$1,$3); } /* | term '&' factor */ | factor ; relational_operator: '>' { $$ = ">"; } | EQ { $$ = "="; } | '<' { $$ = "<"; } | GE { $$ = ">="; } | LE { $$ = "<="; } | NE { $$ = "<>"; } ; condition : term relational_operator term relational_operator term { $$ = FunctionFactory::instance().build("and", FunctionFactory::instance().build($2,$1,$3), FunctionFactory::instance().build($4,$3,$5)); } | term relational_operator term { $$ = FunctionFactory::instance().build($2, $1, $3); } | MATCH '(' expression_list ')' IN QUERY '(' select_statement ')' { NOTIMP; // const Expressions& matchList ($3); // const SelectAST& subquery ($8); // $$ = FunctionFactory::instance().build("match", matchList, subquery); } | condition IN '(' expression_list ')' { $4.push_back($1); $$ = FunctionFactory::instance().build("in",$4); } | condition NOT IN '(' expression_list ')' { $5.push_back($1); $$ = FunctionFactory::instance().build("not_in",$5); } | NOT condition { $$ = FunctionFactory::instance().build("not",$2); } | condition IS NIL { $$ = FunctionFactory::instance().build("null",$1); } | condition IS NOT NIL { $$ = FunctionFactory::instance().build("not_null",$1); } | condition BETWEEN term AND term { $$ = FunctionFactory::instance().build("between",$1,$3,$5); } | condition NOT BETWEEN term AND term { $$ = FunctionFactory::instance().build("not_between",$1,$4,$6); } | condition LIKE term { $$ = FunctionFactory::instance().build("like", $1, $3); } | condition RLIKE term { $$ = FunctionFactory::instance().build("rlike", $1, $3); } | term ; conjonction : conjonction AND condition { $$ = FunctionFactory::instance().build("and",$1,$3); } | condition ; disjonction : disjonction OR conjonction { $$ = FunctionFactory::instance().build("or",$1,$3); } | conjonction ; expression : disjonction | expression '[' expression ']' { // This has not been implemented yet. // SDS: n.b. seems to be in old versions. Not sure why deprecated throw UserError("Syntax: 'expression [ expression ]' not yet supported"); } ; empty : ; %% #undef YY_NULL #include "sqll.c" eckit-2.0.7/src/eckit/sql/SQLSelectFactory.h0000664000175000017500000000554715161702250021002 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_sql_SQLSelectFactory_H #define eckit_sql_SQLSelectFactory_H #include #include #include "eckit/sql/SQLOutputConfig.h" // Forward declarations namespace eckit { class DataHandle; namespace sql { class SQLDatabase; class SQLSession; class SQLTable; class SQLSelect; class SQLOutput; namespace expression { class Expressions; class SQLExpression; } // namespace expression } // namespace sql } // namespace eckit namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLSelectFactory { public: SQLSelectFactory(SQLSession& session); SQLSelect* create(bool distinct, const expression::Expressions& select_list, const std::string& into, // n.b. not const SQLTable only for ease of integration with sqly.y const std::vector>& from, std::shared_ptr where, const expression::Expressions& group_by, std::pair> order_by); std::shared_ptr createColumn(const std::string& columnName, const std::string& bitfieldName, std::shared_ptr& vectorIndex, const std::string& tableReference, std::shared_ptr& pshift); SQLDatabase& database() { return database_; } // static odb::MetaData toODAColumns(odb::sql::SQLSession&, const odb::sql::TableDef&); private: // methods // No copy allowed SQLSelectFactory(const SQLSelectFactory&); SQLSelectFactory& operator=(const SQLSelectFactory&); std::string index(const std::string& columnName, const expression::SQLExpression* index); private: // members SQLSession& session_; // SchemaAnalyzer& analyzer(); // MetaData columns(const std::string& tableName); SQLDatabase& database_; SQLOutputConfig config_; int maxColumnShift_; int minColumnShift_; // friend class eckit::NewAlloc0; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLDatabase.h0000664000175000017500000000675415161702250017740 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @author Baudouin Raoult /// @date Dec 2003 #ifndef eckit_sql_SQLDatabase_H #define eckit_sql_SQLDatabase_H #include #include #include "eckit/filesystem/PathName.h" #include "eckit/sql/SQLTable.h" #include "eckit/sql/SchemaAnalyzer.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- // Forward declarations class SQLStatement; namespace expression { class SQLExpression; } using Links = std::map>; using Variables = std::map>; //---------------------------------------------------------------------------------------------------------------------- class SQLDatabase { public: SQLDatabase(const std::string& name = "default"); virtual ~SQLDatabase(); SQLDatabase(const SQLDatabase&) = delete; SQLDatabase& operator=(const SQLDatabase&) = delete; SQLDatabase(SQLDatabase&&) = default; SQLDatabase& operator=(SQLDatabase&&) = default; // -- Methods void open(); void close(); /// Access SQLTables in DB bool hasTable(const std::string& name) const; SQLTable& table(const std::string& name); SQLTable& defaultTable(); std::vector> implicitTables(); // SQLTable& openDataHandle(eckit::DataHandle&); // SQLTable& openDataStream(std::istream&, const std::string& delimiter); void addTable(SQLTable* table); void addImplicitTable(SQLTable* table); void setLinks(const Links&); void setLinks() { setLinks(links_); } void addLinks(const Links& ls) { links_.insert(ls.begin(), ls.end()); } Links& links() { return links_; } const std::string& name() const { return name_; } std::shared_ptr getVariable(const std::string&) const; void setVariable(const std::string&, std::shared_ptr); Variables& variables() { return variables_; } SchemaAnalyzer& schemaAnalyzer() { return schemaAnalyzer_; } void setIncludePath(const std::string& includePath); const std::vector& includePath() const { return includePath_; } protected: Links links_; std::map> tablesByName_; std::vector> implicitTables_; std::vector includePath_; Variables variables_; std::string name_; SchemaAnalyzer schemaAnalyzer_; private: // No copy allowed void loadIOMAP(); void loadDD(); void loadFLAGS(); // -- Friends friend std::ostream& operator<<(std::ostream& s, const SQLDatabase& p) { s << "[SQLDatabase@" << &p << " tables: "; for (const auto& it : p.tablesByName_) { s << it.first << ","; } s << "]"; return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif // eckit_sql_SQLDatabase_H eckit-2.0.7/src/eckit/sql/SQLOrderOutput.cc0000664000175000017500000001056015161702250020654 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLOrderOutput.h" #include "eckit/sql/expression/SQLExpressionEvaluated.h" using namespace eckit::sql::expression; /// @note This code triggers a segmentation fault on the CRAY CC compiler 8.4 when optimisation is turned on /// The compiler bug has been fixed in versions >= 8.5 /// We have chosen to turn off optimisations for all CRAY compilers below, since this is not perfromance critical /// See ECKIT-405 ECKIT-406 #ifdef _CRAYCC #pragma _CRI noopt #endif namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SQLOrderOutput::SQLOrderOutput(SQLOutput& output, const std::pair>& by) : output_(output), by_(by) {} SQLOrderOutput::~SQLOrderOutput() {} void SQLOrderOutput::print(std::ostream& s) const { s << "SQLOrderOutput[" << output_ << " ORDER BY "; for (size_t i = 0; i < by_.first.size(); i++) { s << *(by_.first[i]) << (by_.second[i] ? " ASC " : " DESC ") << ", "; } s << "]"; } unsigned long long SQLOrderOutput::count() { return output_.count(); } void SQLOrderOutput::reset() { output_.reset(); } void SQLOrderOutput::flush() { output_.flush(); } bool SQLOrderOutput::cachedNext() { while (true) { auto it = sortedResults_.begin(); // If there are no more results, we are done if (it == sortedResults_.end()) { return false; } // Given identical sorted keys, we use the order that rows are appended std::queue& rows = it->second; ASSERT(rows.size() >= 1); bool success = output_.output(rows.front()); // Remove entries that have been output rows.pop(); if (rows.empty()) { sortedResults_.erase(it); } if (success) { return true; } } } bool SQLOrderOutput::output(const Expressions& results) { OrderByExpressions byValues(by_.second); Expressions& byExpressions(by_.first); for (size_t i = 0; i < byExpressions.size(); ++i) { byValues.push_back( std::make_shared(byIndices_[i] ? *results[byIndices_[i] - 1] : *byExpressions[i])); } Expressions resultValues; for (size_t i = 0; i < results.size(); ++i) { resultValues.push_back(std::make_shared(*results[i])); } sortedResults_[byValues].push(resultValues); return false; } void SQLOrderOutput::preprepare(SQLSelect& sql) { output_.preprepare(sql); for (auto& exprn : by_.first) { exprn->preprepare(sql); } } void SQLOrderOutput::prepare(SQLSelect& sql) { output_.prepare(sql); Expressions& ex(by_.first); for (size_t i(0); i < ex.size(); ++i) { if (!ex[i]->isConstant()) { ex[i]->prepare(sql); byIndices_.push_back(0); } else { bool missing(false); size_t index(ex[i]->eval(missing)); ASSERT(!missing); if (index < 1) { throw eckit::UserError("ORDER BY: indices of columns must be positive"); } byIndices_.push_back(index); } } } void SQLOrderOutput::cleanup(SQLSelect& sql) { output_.cleanup(sql); for (Expressions::iterator j = by_.first.begin(); j != by_.first.end(); ++j) { (*j)->cleanup(sql); } } // Direct output functions removed in order output void SQLOrderOutput::outputReal(double, bool) { NOTIMP; } void SQLOrderOutput::outputDouble(double, bool) { NOTIMP; } void SQLOrderOutput::outputInt(double, bool) { NOTIMP; } void SQLOrderOutput::outputUnsignedInt(double, bool) { NOTIMP; } void SQLOrderOutput::outputString(const char*, size_t, bool) { NOTIMP; } void SQLOrderOutput::outputBitfield(double, bool) { NOTIMP; } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SchemaAnalyzer.h0000664000175000017500000000445315161702250020554 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Piotr Kuchta /// @author Simon Smart /// date April 2009 /// date January 2019 #ifndef eckit_sql_SchemaAnalyzer_H #define eckit_sql_SchemaAnalyzer_H #include #include "eckit/sql/SQLTypedefs.h" #include "eckit/sql/SchemaComponents.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SchemaAnalyzer { public: // methods SchemaAnalyzer(); ~SchemaAnalyzer(); void addTable(TableDef& table); void addBitfieldType(const std::string& name, const FieldNames& fields, const Sizes& sizes); //, const std::string& typeSignature); const BitfieldDef& getBitfieldType(const std::string& typeName) const; const BitfieldDef& getBitfieldTypeDefinition(const std::string& columnName) const; std::string generateSelectAll(const std::set& skipTables = std::set()) const; TableDefs definitions() const; // bool isBitfield(const std::string& columnName) const; // const BitfieldDef& getBitfieldTypeDefinition(const std::string& columnName); // bool tableKnown(const std::string& name) const; // const TableDef& findTable(const std::string& name) const; // void skipTable(std::string tableName); // Definitions generateDefinitions(); // std::string findColumnType(const std::string&); private: BitfieldDefs bitfieldTypes_; TableDefs tableDefs_; std::map columnTypes_; // std::string currentSchema_; // SchemaDefs schemas_; // TableDefs tableDefs_; // BitfieldDefs bitfieldTypes_; // std::set tablesToSkip_; // std::map columnTypes_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLOutputConfig.h0000664000175000017500000001072315161702250020651 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// ECMWF Jul 2010 #ifndef eckit_sql_SQLOutputConfig_H #define eckit_sql_SQLOutputConfig_H #include "eckit/filesystem/PathName.h" namespace eckit::sql { class SQLOutput; //---------------------------------------------------------------------------------------------------------------------- // SQLOutputConfig provides a mechanism for an application to construct output functionality // according to its requirements: // // i) User specification (e.g. command line options) // ii) Application requirements (e.g. output file format) // iii) SQL statements (i.e. INTO, which cannot be specified before the SQL command is run). class SQLOutputConfig { public: // methods SQLOutputConfig(bool noColumnNames = false, bool noNULL = false, const std::string& delimiter = defaultDelimiter, const std::string& format = defaultOutputFormat, bool bitfieldsBinary = false, bool noColumnAlignment = false, bool fullPrecision = false); virtual ~SQLOutputConfig(); SQLOutput* buildOutput() const; virtual SQLOutput* buildOutput(const eckit::PathName& path) const; void setOutputFile(const eckit::PathName& filename); // Standard format properties (may not be applicable to all output types, but needed for // compatibility with SQLSimpleOutput). const std::string& fieldDelimiter() const; const std::string& outputFormat() const; bool doNotWriteNULL() const; bool fullPrecision() const; bool displayBitfieldsBinary() const; bool disableAlignmentOfColumns() const; bool doNotWriteColumnNames() const; // Defaults! static const char* defaultDelimiter; static const char* defaultOutputFormat; protected: // members eckit::PathName outputFile_; bool doNotWriteColumnNames_; std::string fieldDelimiter_; std::string outputFormat_; bool displayBitfieldsBinary_; // --binary bool disableAlignmentOfColumns_; // --no_alignment bool fullPrecision_; // --full_precision bool doNotWriteNULL_; }; // TODO: Move into odc // SQLOutputConfig(bool cn = false, // bool n = false, // const std::string& d = defaultDelimiter(), // const std::string& output = defaultOutputFile(), // const std::string& format = defaultFormat(), // bool displayBitfieldsBinary = false, // bool displayBitfieldsHexadecimal = false, // bool disableAlignmentOfColumns = false, // bool fullPrecision = false); // // bool doNotWriteColumnNames () const; // void doNotWriteColumnNames(bool b); // // bool doNotWriteNULL () const; // void doNotWriteNULL (bool b); // // void fieldDelimiter(const std::string& d); // // const std::string& outputFile () const; // void outputFile (const std::string& fn); // // const std::string& outputFormat () const; // void outputFormat (const std::string& s); // // bool displayBitfieldsBinary () const; // void displayBitfieldsBinary (bool b); // // bool displayBitfieldsHexadecimal () const; // void displayBitfieldsHexadecimal (bool b); // // bool disableAlignmentOfColumns () const; // void disableAlignmentOfColumns (bool b); // // bool fullPrecision () const; // void fullPrecision (bool); // // static const SQLOutputConfig defaultConfig(); // // private: // bool doNotWriteColumnNames_; // bool doNotWriteNULL_; // std::string fieldDelimiter_; // std::string outputFile_; // -o // std::string outputFormat_; // -f // bool displayBitfieldsBinary_; // --binary // bool displayBitfieldsHexadecimal_; // --hex // bool disableAlignmentOfColumns_; // --no_alignment // bool fullPrecision_; // --full_precision // std::string outputFormat_; // -f // bool displayBitfieldsHexadecimal_; // --hex // // }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLTableFactory.h0000664000175000017500000000503015161702250020575 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @date Aug 2018 #ifndef eckit_sql_SQLTableFactory_H #define eckit_sql_SQLTableFactory_H #include #include #include /// Depending on what is being used, we need to be able to open SQL tables of different /// sorts from different locations. /// /// --> The SQLParser/SQLSession can see a select statement such as "select a from b.tbl" /// and look for the handler of the file b.tbl /// /// TODO: Consider whether this really ought to be a global factory, or something that /// is handled on a per session basis. namespace eckit::sql { class SQLTable; class SQLTableFactoryBase; class SQLDatabase; //---------------------------------------------------------------------------------------------------------------------- class SQLTableFactory { // n.b. we currently have no use for a name, but we may well in the future. // using factory_map = std::map; using factory_map = std::vector; private: // methods // Can only be constructed by instance() SQLTableFactory() = default; ~SQLTableFactory() = default; public: // methods static SQLTableFactory& instance(); /// Build an SQLTable from a give name. Location is optional - if it is not supplied /// then the name may be treated as a location at the discretion of the handler. SQLTable* build(SQLDatabase& owner, const std::string& name, const std::string& location = ""); void enregister(SQLTableFactoryBase* f); void deregister(SQLTableFactoryBase* f); private: // methods factory_map factories_; std::mutex mutex_; }; //---------------------------------------------------------------------------------------------------------------------- class SQLTableFactoryBase { public: // methods SQLTableFactoryBase(); virtual ~SQLTableFactoryBase(); virtual SQLTable* build(SQLDatabase& owner, const std::string& name, const std::string& location) const = 0; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLOutputConfig.cc0000664000175000017500000000540115161702250021004 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLOutputConfig.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/sql/SQLSimpleOutput.h" #include "eckit/os/BackTrace.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- SQLOutputConfig::SQLOutputConfig(bool noColumnNames, bool noNULL, const std::string& delimiter, const std::string& format, bool bitfieldsBinary, bool noColumnAlignment, bool fullPrecision) : outputFile_(""), doNotWriteColumnNames_(noColumnNames), fieldDelimiter_(delimiter), outputFormat_(format), displayBitfieldsBinary_(bitfieldsBinary), disableAlignmentOfColumns_(noColumnAlignment), fullPrecision_(fullPrecision), doNotWriteNULL_(noNULL) {} SQLOutputConfig::~SQLOutputConfig() {} void SQLOutputConfig::setOutputFile(const eckit::PathName& filename) { outputFile_ = filename; } SQLOutput* SQLOutputConfig::buildOutput() const { return buildOutput(outputFile_); } SQLOutput* SQLOutputConfig::buildOutput(const eckit::PathName& path) const { // Output file not supported in base config ASSERT(path.asString().empty()); if (outputFormat_ != "default" && outputFormat_ != "wide") { throw UserError("Unsupported output format: " + outputFormat_, Here()); } return new SQLSimpleOutput(*this, std::cout); } const std::string& SQLOutputConfig::fieldDelimiter() const { return fieldDelimiter_; } const std::string& SQLOutputConfig::outputFormat() const { return outputFormat_; } bool SQLOutputConfig::doNotWriteNULL() const { return doNotWriteNULL_; } bool SQLOutputConfig::fullPrecision() const { return fullPrecision_; } bool SQLOutputConfig::displayBitfieldsBinary() const { return displayBitfieldsBinary_; } bool SQLOutputConfig::disableAlignmentOfColumns() const { return disableAlignmentOfColumns_; } bool SQLOutputConfig::doNotWriteColumnNames() const { return doNotWriteColumnNames_; } const char* SQLOutputConfig::defaultDelimiter = " "; // const char* SQLOutputConfig::defaultOutputFile = "output.odb"; const char* SQLOutputConfig::defaultOutputFormat = "default"; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLSimpleOutput.h0000664000175000017500000000417715161702250020703 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 2003 #ifndef eckit_sql_SQLSimpleOutput_H #define eckit_sql_SQLSimpleOutput_H #include "eckit/sql/SQLOutput.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLSimpleOutput : public SQLOutput { public: // methods SQLSimpleOutput(const SQLOutputConfig& config, std::ostream& out); ~SQLSimpleOutput() override; private: // methods template void outputValue(double x, bool missing); std::ostream& format(std::ostream&, size_t) const; void printHeader(SQLSelect&); private: // methods (overrides) void print(std::ostream&) const override; void reset() override; void flush() override; bool output(const expression::Expressions&) override; void prepare(SQLSelect&) override; void updateTypes(SQLSelect&) override; void cleanup(SQLSelect&) override; unsigned long long count() override; void outputReal(double, bool) override; void outputDouble(double, bool) override; void outputInt(double, bool) override; void outputUnsignedInt(double, bool) override; void outputString(const char*, size_t, bool) override; void outputBitfield(double, bool) override; private: // members std::ostream& out_; unsigned long long count_; std::vector columnWidths_; using manipulator = std::ios_base& (*)(std::ios_base&); std::vector columnAlignments_; size_t currentColumn_; const SQLOutputConfig& config_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLTable.cc0000664000175000017500000001775415161702250017423 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLTable.h" #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLBitColumn.h" #include "eckit/sql/SQLColumn.h" #include "eckit/sql/SQLDatabase.h" #include "eckit/utils/Tokenizer.h" namespace eckit::sql { SQLTable::SQLTable(SQLDatabase& owner, const std::string& path, const std::string& name) : path_(path), name_(name), owner_(owner) { Log::debug() << "new SQLTable[path=" << path_ << ",name=" << name << "]" << std::endl; } SQLTable::~SQLTable() { clearColumns(); } void SQLTable::clearColumns() {} std::vector SQLTable::columnNames() const { std::vector results; for (std::map::const_iterator j = columnsByIndex_.begin(); j != columnsByIndex_.end(); ++j) { results.push_back((*j).second->name()); } return results; } FieldNames SQLTable::bitColumnNames(const std::string& name) const { using I = std::map::const_iterator; I i = bitColumnNames_.find(name); if (i != bitColumnNames_.end()) { return (*i).second; } ASSERT("name not found" && name.find("@") == std::string::npos); std::string columnName; FieldNames fieldNames; size_t counter = 0; for (I i = bitColumnNames_.begin(); i != bitColumnNames_.end(); ++i) { if (i->first.find(name + "@") == 0) { columnName = i->first; fieldNames = i->second; ++counter; } } if (counter == 0) { throw eckit::UserError(std::string("Column '") + name + "' not found."); } if (counter != 1) { throw eckit::UserError(std::string("Ambiguous column name: '") + name + "'"); } return fieldNames; } // void SQLTable::addColumn(const std::string& name, int index, const type::SQLType& type, const FieldNames& bitmap) void SQLTable::addColumn(const std::string& name, int index, const type::SQLType& type, bool hasMissingValue, double missingValue, bool isBitfield, const BitfieldDef& bitfieldDef) { const FieldNames& bitmap = bitfieldDef.first; std::unique_ptr col(isBitfield ? createSQLColumn(type, name, index, hasMissingValue, missingValue, bitfieldDef) : createSQLColumn(type, name, index, hasMissingValue, missingValue)); // ownedColumns_ describes columns for deletion. Once we sort out the bitfield/SQLBitColumn // ambiguity in this file, ownedColumns_ should hopefully become superfluous, and // columnsByName_ can do the owning. columnsByName_[name] = col.get(); columnsByIndex_[index] = col.get(); bitColumnNames_[name] = bitmap; std::vector tokens; Tokenizer("@")(name, tokens); ASSERT(tokens.size() == 1 || tokens.size() == 2); // TODO: clean up, probably no need to do this parsing as we have the whole bitfieldDef now std::string tableName = (tokens.size() == 2) ? tokens[1] : ""; std::string columnName = tokens[0]; for (FieldNames::const_iterator j = bitmap.begin(); j != bitmap.end(); ++j) { std::string fieldName = *j; std::string n = columnName + "." + fieldName; if (!tableName.empty()) { n += "@" + tableName; } columnsByName_[n] = col.get(); // Log::info() << "SQLTable::addColumn: columnsByName_[" << n << "] = " << *col << std::endl; } ownedColumns_.push_back(std::move(col)); } void SQLTable::addColumn(SQLColumn* col, const std::string& name, int index) { columnsByName_[name] = col; columnsByIndex_[index] = col; } SQLColumn* SQLTable::createSQLColumn(const type::SQLType& type, const std::string& name, size_t index, bool hasMissingValue, double missingValue, const BitfieldDef& defs) { ASSERT(type.size() % 8 == 0); return new SQLColumn(type, *this, name, index, hasMissingValue, missingValue, defs); } bool SQLTable::hasColumn(const std::string& name) const { std::map::const_iterator j = columnsByName_.find(name); return j != columnsByName_.end(); } unsigned long long SQLTable::noRows() const { std::map::const_iterator j = columnsByName_.begin(); if (j != columnsByName_.end()) { return (*j).second->noRows(); } return 0; } const SQLColumn& SQLTable::column(const std::string& name) const { std::map::const_iterator j = columnsByName_.find(name); if (j != columnsByName_.end()) { return *(j->second); } std::vector v; Tokenizer(".")(name, v); // *** HACK alert *** // TODO: this does not seem to be the correct place to construct SQLBitColumns // If we are referring to a bit column, construct it here. This is essentially a // "caching" trick, so it is sorta-ok that we mutate state despite this function // being logically const... // TODO: The combination of this and SQLTable::addColumn() construction seems duplicative // something is not right, and one of these should be removed. if (v.size() > 1) { if (hasColumn(v[0])) { const SQLColumn& col = column(v[0]); SQLTable* mutableThis = const_cast(this); std::unique_ptr newCol(new SQLBitColumn(col, v[1])); SQLColumn* pnewCol = newCol.get(); mutableThis->columnsByName_[name] = pnewCol; mutableThis->ownedColumns_.push_back(std::move(newCol)); return *pnewCol; } } // *** End HACK **** throw eckit::UserError("Column not found", name); } void SQLTable::updateColumnDoublesWidth(const std::string& name, size_t nDoubles) { std::map::const_iterator j = columnsByName_.find(name); if (j == columnsByName_.end()) { throw eckit::UserError("Column not found", name); } if (j->second->type().getKind() == type::SQLType::stringType) { j->second->updateType(type::SQLType::lookup("string", nDoubles)); } else { ASSERT(nDoubles == 1); } } void SQLTable::updateColumnMissingValues(const std::string& name, bool hasMissing, double missingValue) { std::map::const_iterator j = columnsByName_.find(name); if (j == columnsByName_.end()) { throw eckit::UserError("Column not found", name); } j->second->hasMissingValue(hasMissing); j->second->missingValue(missingValue); } void SQLTable::addLinkFrom(const SQLTable& from) { linksFrom_.insert(from); } bool SQLTable::hasLinkFrom(const SQLTable& from) const { // return linksFrom_.find(from) != linksFrom_.end(); return false; } void SQLTable::addLinkTo(const SQLTable& to) { linksTo_.insert(to); } bool SQLTable::hasLinkTo(const SQLTable& to) const { // return linksTo_.find(to) != linksTo_.end(); return false; } bool SQLTable::isParentOf(const SQLTable& other) const { if (hasLinkTo(other)) { return true; } for (const auto& l : linksTo_) { if (l.get().isParentOf(other)) { return true; } } return false; } std::string SQLTable::fullName() const { return owner_.name() + "." + name_; } void SQLTable::print(std::ostream& s) const { s << "CREATE TABLE " << fullName() << " AS (" << std::endl; for (std::map::const_iterator j = columnsByIndex_.begin(); j != columnsByIndex_.end(); ++j) { SQLColumn* c = j->second; s << " " << c->name() << " " << c->type() << "," << std::endl; } s << ")" << std::endl; } } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/SQLTypedefs.h0000664000175000017500000000142115161702250020001 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef eckit_sql_SQLTypes_h #define eckit_sql_SQLTypes_h #include #include #include #include namespace eckit::sql { using FieldNames = std::vector; using Sizes = std::vector; using BitfieldDef = std::pair; using BitfieldDefs = std::map; } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/CMakeLists.txt0000664000175000017500000001321115161702250020225 0ustar alastairalastairlist( APPEND eckit_sql_lib_srcs Environment.cc Environment.h SQLBitColumn.cc SQLBitColumn.h SQLColumn.cc SQLColumn.h SQLDatabase.cc SQLDatabase.h SQLDistinctOutput.cc SQLDistinctOutput.h SQLOrderOutput.cc SQLOrderOutput.h SQLOutput.cc SQLOutput.h SQLOutputConfig.cc SQLOutputConfig.h SQLParser.cc SQLParser.h SQLSelect.cc SQLSelect.h SQLSelectFactory.cc SQLSelectFactory.h SQLSession.cc SQLSession.h SQLSimpleOutput.cc SQLSimpleOutput.h SQLStatement.cc SQLStatement.h SQLTable.cc SQLTable.h SQLTableFactory.cc SQLTableFactory.h SQLTypedefs.h SchemaAnalyzer.cc SchemaAnalyzer.h SchemaComponents.cc SchemaComponents.h SelectOneTable.cc SelectOneTable.h expression/BitColumnExpression.cc expression/BitColumnExpression.h expression/ColumnExpression.cc expression/ColumnExpression.h expression/ConstantExpression.cc expression/ConstantExpression.h expression/NumberExpression.cc expression/NumberExpression.h expression/OrderByExpressions.cc expression/OrderByExpressions.h expression/ParameterExpression.cc expression/ParameterExpression.h expression/SQLExpression.cc expression/SQLExpression.h expression/SQLExpressionEvaluated.cc expression/SQLExpressionEvaluated.h expression/SQLExpressions.cc expression/SQLExpressions.h expression/ShiftedColumnExpression.cc expression/ShiftedColumnExpression.h expression/StringExpression.cc expression/StringExpression.h expression/function/DoubleFunctions.cc expression/function/FunctionAND.cc expression/function/FunctionAND.h expression/function/FunctionAVG.cc expression/function/FunctionAVG.h expression/function/FunctionCOUNT.cc expression/function/FunctionCOUNT.h expression/function/FunctionDOTP.cc expression/function/FunctionDOTP.h expression/function/FunctionEQ.cc expression/function/FunctionEQ.h expression/function/FunctionExpression.cc expression/function/FunctionExpression.h expression/function/FunctionFIRST.cc expression/function/FunctionFIRST.h expression/function/FunctionFactory.cc expression/function/FunctionFactory.h expression/function/FunctionIN.cc expression/function/FunctionIN.h expression/function/FunctionIntegerExpression.cc expression/function/FunctionIntegerExpression.h expression/function/FunctionJOIN.cc expression/function/FunctionJOIN.h expression/function/FunctionJULIAN.cc expression/function/FunctionJULIAN.h expression/function/FunctionJULIAN_SECONDS.cc expression/function/FunctionJULIAN_SECONDS.h expression/function/FunctionLAST.cc expression/function/FunctionLAST.h expression/function/FunctionMAX.cc expression/function/FunctionMAX.h expression/function/FunctionMIN.cc expression/function/FunctionMIN.h expression/function/FunctionNE.cc expression/function/FunctionNE.h expression/function/FunctionNORM.cc expression/function/FunctionNORM.h expression/function/FunctionNOT_IN.cc expression/function/FunctionNOT_IN.h expression/function/FunctionNOT_NULL.cc expression/function/FunctionNOT_NULL.h expression/function/FunctionNULL.cc expression/function/FunctionNULL.h expression/function/FunctionNVL.cc expression/function/FunctionNVL.h expression/function/FunctionOR.cc expression/function/FunctionOR.h expression/function/FunctionRLIKE.cc expression/function/FunctionRLIKE.h expression/function/FunctionRMS.cc expression/function/FunctionRMS.h expression/function/FunctionROWNUMBER.cc expression/function/FunctionROWNUMBER.h expression/function/FunctionSTDEV.cc expression/function/FunctionSTDEV.h expression/function/FunctionSUM.cc expression/function/FunctionSUM.h expression/function/FunctionTDIFF.cc expression/function/FunctionTDIFF.h expression/function/FunctionTHIN.cc expression/function/FunctionTHIN.h expression/function/FunctionTIMESTAMP.cc expression/function/FunctionTIMESTAMP.h expression/function/FunctionVAR.cc expression/function/FunctionVAR.h type/SQLBit.cc type/SQLBit.h type/SQLBitfield.cc type/SQLBitfield.h type/SQLDouble.cc type/SQLDouble.h type/SQLInt.cc type/SQLInt.h type/SQLReal.cc type/SQLReal.h type/SQLString.cc type/SQLString.h type/SQLType.cc type/SQLType.h ) ecbuild_generate_yy( YYPREFIX eckit_sql_ YACC sqly LEX sqll FLEX_FLAGS "-d" DEPENDANT SQLParser.cc ) # Supress warnings emited from YACC/LEX generated code if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") list(APPEND suppress_warnings -Wno-unused-function ) elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") list(APPEND suppress_warnings -Wno-unused-function -Wno-unused-but-set-variable ) endif() set_source_files_properties( SQLParser.cc PROPERTIES COMPILE_OPTIONS "${suppress_warnings}" ) ecbuild_add_library( TARGET eckit_sql TYPE SHARED INSTALL_HEADERS ALL HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/eckit/sql SOURCES ${eckit_sql_lib_srcs} TEMPLATES SQLIteratorOutput.cc expression/ShiftedColumnExpression.cc PUBLIC_LIBS eckit ) # Disable all warnings for PGI/NVHPC compiler as it is overzealous and cherry-picking is not possible target_compile_options( eckit_sql PRIVATE $<$:-w> $<$:-w> ) eckit-2.0.7/src/eckit/sql/SQLSession.h0000664000175000017500000000706315161702250017651 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Baudouin Raoult /// @author Simon Smart /// @date Dec 03 #ifndef odb_sql_SQLSession_H #define odb_sql_SQLSession_H namespace eckit { class PathName; } namespace eckit { class DataHandle; } #include #include "eckit/sql/SQLSelectFactory.h" // #include "eckit/sql/SQLInsertFactory.h" #include "eckit/sql/SQLDatabase.h" #include "eckit/sql/SQLOutputConfig.h" namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLOutput; class SQLDatabase; class SQLStatement; class SQLTable; class SQLOutputConfig; class SQLSession { public: // Constructors SQLSession(std::unique_ptr out, std::unique_ptr config = nullptr, const std::string& csvDelimiter = ","); SQLSession(std::unique_ptr config, const std::string& csvDelimiter = ","); SQLSession(std::unique_ptr out, const std::string& csvDelimiter); SQLSession(const std::string& csvDelimiter = ","); SQLSession(const SQLSession&) = delete; SQLSession& operator=(const SQLSession&) = delete; SQLSession(SQLSession&&) = delete; SQLSession& operator=(SQLSession&&) = delete; virtual ~SQLSession(); // For sqly.y (used parsing SQL strings) virtual SQLSelectFactory& selectFactory(); // virtual SQLInsertFactory& insertFactory(); virtual SQLTable& findTable(const std::string& name); // virtual SQLTable* openDataHandle(eckit::DataHandle &); // virtual SQLTable* openDataStream(std::istream &, const std::string &); virtual void setStatement(SQLStatement*); virtual SQLStatement& statement(); virtual SQLOutput& output(); virtual const SQLDatabase& currentDatabase() const; virtual SQLDatabase& currentDatabase(); virtual unsigned long long execute(SQLStatement&); virtual void interactive() {} unsigned long long lastExecuteResult() { return lastExecuteResult_; } std::string csvDelimiter() { return csvDelimiter_; } // const SQLOutputConfig& outputConfig() { ASSERT(config_); return *config_; } std::unique_ptr newFileOutput(const eckit::PathName& path); static std::string readIncludeFile(const std::string&); protected: void loadDefaultSchema(); private: static std::string schemaFile(); static std::vector includePaths(); SQLDatabase database_; // std::map params_; // std::map databases_; SQLSelectFactory selectFactory_; // SQLInsertFactory insertFactory_; unsigned long long lastExecuteResult_; std::unique_ptr config_; std::unique_ptr statement_; std::unique_ptr output_; const std::string csvDelimiter_; friend std::ostream& operator<<(std::ostream& s, const SQLSession& p) { s << "[session@" << &p << ", currentDatabase: " << p.currentDatabase() << " ]"; return s; } }; //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/SQLColumn.cc0000664000175000017500000000526415161702250017622 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLColumn.h" #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLTable.h" using namespace eckit; namespace eckit::sql { SQLColumn::SQLColumn(const type::SQLType& type, SQLTable& owner, const std::string& name, size_t index, bool hasMissingValue, double missingValue, const BitfieldDef& bitfieldDef) : SQLIterator(type), noRows_(0), owner_(owner), name_(name), index_(index), current_(0), last_(0), position_(0), iterator_{nullptr}, hasMissingValue_(hasMissingValue), missingValue_(missingValue), isBitfield_(true), bitfieldDef_(bitfieldDef), sizeDoubles_(type.size() / 8) { ASSERT(type.size() % 8 == 0); } SQLColumn::SQLColumn(const SQLColumn& other) : SQLIterator(other.type()), noRows_(0), owner_(other.owner_), name_(other.name_), index_(other.index_), current_(0), last_(0), position_(0), iterator_{nullptr}, hasMissingValue_(other.hasMissingValue_), missingValue_(other.missingValue_), isBitfield_(other.isBitfield_), bitfieldDef_(other.bitfieldDef_), sizeDoubles_(other.sizeDoubles_) { ASSERT(type_.get().size() % 8 == 0); } SQLColumn::~SQLColumn() {} void SQLColumn::rewind() { // Timer timer("SQLColumn::rewind"); rows_.clear(); iterators_.clear(); noRows_ = 0; // setPool(0); } void SQLColumn::setPool(int n) { if (iterator_) { iterator_->unload(); } current_ = n; position_ = 0; last_ = rows_[n]; iterator_ = iterators_[n]; iterator_->rewind(); // cout << "pool " << n << std::endl; // cout << "pool " << n << " " << last_ << std::endl; } double SQLColumn::next(bool& missing) { if (position_ == last_) { setPool(current_ + 1); } position_++; return iterator_->next(missing); } void SQLColumn::advance(unsigned long) { NOTIMP; } unsigned long long SQLColumn::noRows() const { if (!noRows_) { SQLColumn* col = const_cast(this); col->rewind(); } return noRows_; } void SQLColumn::print(std::ostream& s) const {} std::string SQLColumn::fullName() const { // FIXME: return name(); // + "@" + owner_.fullName(); } SQLTable* SQLColumn::table() const { return &owner_; } } // namespace eckit::sql eckit-2.0.7/src/eckit/sql/type/0000775000175000017500000000000015161702250016450 5ustar alastairalastaireckit-2.0.7/src/eckit/sql/type/SQLDouble.cc0000664000175000017500000000223615161702250020554 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/type/SQLDouble.h" #include "eckit/sql/SQLOutput.h" #include "eckit/utils/Translator.h" namespace eckit::sql::type { //---------------------------------------------------------------------------------------------------------------------- SQLDouble::SQLDouble(const std::string& name) : SQLType(name) {} SQLDouble::~SQLDouble() {} size_t SQLDouble::size() const { return sizeof(double); } void SQLDouble::output(SQLOutput& o, double d, bool m) const { o.outputDouble(d, m); } std::string SQLDouble::asString(const double* val) const { return eckit::Translator()(*val); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::type eckit-2.0.7/src/eckit/sql/type/SQLReal.h0000664000175000017500000000352615161702250020072 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLReal.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLReal_H #define SQLReal_H #include "eckit/sql/type/SQLType.h" namespace eckit::sql { class SQLOutput; namespace type { class SQLReal : public SQLType { public: // -- Exceptions // None // -- Contructors SQLReal(const std::string&); // -- Destructor ~SQLReal(); // -- Convertors // None // -- Operators // None // -- Methods // None // -- Overridden methods using SQLType::output; void output(SQLOutput&, double, bool) const override; // -- Class members // None // -- Class methods // None protected: // -- Members // None // -- Methods // void print(std::ostream&) const; // -- Overridden methods // None // -- Class members // None // -- Class methods // None private: // No copy allowed SQLReal(const SQLReal&); SQLReal& operator=(const SQLReal&); // -- Members // None // -- Methods // None // -- Overridden methods // None size_t size() const override; int getKind() const override { return realType; } std::string asString(const double* val) const override; // -- Class members // None // -- Class methods // None // -- Friends // friend std::ostream& operator<<(std::ostream& s,const SQLReal& p) // { p.print(s); return s; } }; } // namespace type } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/type/SQLType.cc0000664000175000017500000000772215161702250020270 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/type/SQLType.h" #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/sql/type/SQLDouble.h" #include "eckit/sql/type/SQLInt.h" #include "eckit/sql/type/SQLReal.h" #include "eckit/sql/type/SQLString.h" #include "eckit/thread/ThreadSingleton.h" #include "eckit/utils/Translator.h" using namespace eckit; namespace eckit::sql::type { //---------------------------------------------------------------------------------------------------------------------- class TypeRegistry { public: TypeRegistry(); ~TypeRegistry() = default; static TypeRegistry& instance(); // enregister takes ownership void enregister(SQLType* t); void registerAlias(const std::string& name, const std::string& alias); const SQLType* lookup(const std::string& name); private: void enregisterInternal(SQLType* t); std::mutex m_; std::map> map_; std::map aliases_; }; //---------------------------------------------------------------------------------------------------------------------- TypeRegistry::TypeRegistry() { enregisterInternal(new SQLInt("integer")); enregisterInternal(new SQLReal("real")); enregisterInternal(new SQLDouble("double")); } TypeRegistry& TypeRegistry::instance() { static TypeRegistry theRegistry; return theRegistry; } void TypeRegistry::enregister(SQLType* t) { std::lock_guard lock(m_); enregisterInternal(t); } void TypeRegistry::enregisterInternal(SQLType* t) { map_.emplace(std::make_pair(t->name(), std::unique_ptr(t))); } void TypeRegistry::registerAlias(const std::string& name, const std::string& alias) { std::lock_guard lock(m_); auto it = map_.find(name); ASSERT(it != map_.end()); aliases_.emplace(std::make_pair(alias, it->second.get())); } const SQLType* TypeRegistry::lookup(const std::string& name) { std::lock_guard lock(m_); auto it = map_.find(name); if (it != map_.end()) { return it->second.get(); } auto it2 = aliases_.find(name); if (it2 != aliases_.end()) { return it2->second; } return nullptr; } SQLType::SQLType(const std::string& name) : name_(name) {} SQLType::~SQLType() {} size_t SQLType::width() const { return 14; } SQLType::manipulator SQLType::format() const { return &std::right; } bool SQLType::exists(const std::string& name) { const SQLType* typ = TypeRegistry::instance().lookup(name); return !!typ; } const SQLType& SQLType::lookup(const std::string& name, size_t sizeDoubles) { std::string lookupName = name; if (name == "string") { lookupName += Translator()(sizeof(double) * sizeDoubles); } else { ASSERT(sizeDoubles == 1); } const SQLType* typ = TypeRegistry::instance().lookup(lookupName); if (!typ && name == "string") { typ = SQLType::registerType(new SQLString(lookupName, sizeof(double) * sizeDoubles)); } if (!typ) { throw eckit::SeriousBug(name + ": type not defined"); } return *typ; } void SQLType::createAlias(const std::string& name, const std::string& alias) { TypeRegistry::instance().registerAlias(name, alias); } SQLType* SQLType::registerType(SQLType* t) { TypeRegistry::instance().enregister(t); return t; } void SQLType::print(std::ostream& s) const { s << name_; } const SQLType* SQLType::subType(const std::string&) const { return this; } } // namespace eckit::sql::type eckit-2.0.7/src/eckit/sql/type/SQLBitfield.h0000664000175000017500000000363615161702250020733 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLBitfield.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLBitfield_H #define SQLBitfield_H #include "eckit/sql/SQLTypedefs.h" #include "eckit/sql/type/SQLType.h" namespace eckit::sql::type { class SQLBitfield : public SQLType { public: SQLBitfield(const std::string&, const FieldNames&, const Sizes&); ~SQLBitfield(); unsigned long mask(const std::string& n) const; unsigned long shift(const std::string& n) const; const BitfieldDef& bitfieldDef() const { return bitfieldDef_; } const FieldNames& fields() const { return bitfieldDef_.first; } const Sizes& sizes() const { return bitfieldDef_.second; } static std::string make(const std::string&, const FieldNames&, const Sizes&, const char* aliasName = nullptr); private: SQLBitfield(const SQLBitfield&); SQLBitfield& operator=(const SQLBitfield&); static unsigned long makeMask(unsigned long size); BitfieldDef bitfieldDef_; std::map mask_; std::map shift_; size_t size() const override; using SQLType::output; void output(SQLOutput& s, double, bool) const override; std::string asString(const double* val) const override; const SQLType* subType(const std::string&) const override; int getKind() const override { return bitmapType; } size_t width_; size_t width() const override; // friend std::ostream& operator<<(std::ostream& s,const SQLBitfield& p) // { p.print(s); return s; } }; } // namespace eckit::sql::type #endif eckit-2.0.7/src/eckit/sql/type/SQLString.h0000664000175000017500000000255415161702250020455 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLString.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLString_H #define SQLString_H #include "eckit/sql/type/SQLType.h" namespace eckit::sql { class SQLOutput; namespace type { class SQLString : public SQLType { public: SQLString(const std::string& name, size_t maxLen); ~SQLString(); private: // No copy allowed SQLString(const SQLString&); SQLString& operator=(const SQLString&); // -- Overridden methods size_t size() const override; void output(SQLOutput&, double, bool) const override; void output(SQLOutput&, const double*, bool) const override; std::string asString(const double* val) const override; int getKind() const override { return stringType; } manipulator format() const override; size_t width() const override; // friend std::ostream& operator<<(std::ostream& s,const SQLString& p) // { p.print(s); return s; } size_t maxLen_; }; } // namespace type } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/type/SQLBit.cc0000664000175000017500000000224615161702250020061 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/type/SQLBit.h" #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLOutput.h" namespace eckit::sql::type { SQLBit::SQLBit(const std::string& name, unsigned long mask, unsigned long shift) : type::SQLType(name), mask_(mask), shift_(shift) {} SQLBit::~SQLBit() {} // This is an odd one, but it allows us to store values in DISTINCT/ORDER expressions size_t SQLBit::size() const { return sizeof(long); } void SQLBit::output(SQLOutput& o, double x, bool missing) const { double val = (missing ? 0 : ((static_cast(x) & mask_) >> shift_)); o.outputUnsignedInt(val, missing); } std::string SQLBit::asString(const double* val) const { return ((decltype(mask_)(*val) & mask_) == 0) ? "0" : "1"; } } // namespace eckit::sql::type eckit-2.0.7/src/eckit/sql/type/SQLDouble.h0000664000175000017500000000234315161702250020415 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLDouble.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLDouble_H #define SQLDouble_H #include "eckit/sql/type/SQLType.h" namespace eckit::sql { class SQLOutput; namespace type { class SQLDouble : public SQLType { public: SQLDouble(const std::string&); ~SQLDouble(); // -- Overridden methods using SQLType::output; void output(SQLOutput&, double, bool) const override; private: // No copy allowed SQLDouble(const SQLDouble&); SQLDouble& operator=(const SQLDouble&); size_t size() const override; int getKind() const override { return doubleType; } std::string asString(const double* val) const override; // -- Friends // friend std::ostream& operator<<(std::ostream& s,const SQLDouble& p) // { p.print(s); return s; } }; } // namespace type } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/type/SQLBit.h0000664000175000017500000000250715161702250017723 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLBit.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLBit_H #define SQLBit_H #include "eckit/sql/type/SQLType.h" namespace eckit::sql::type { class SQLBit : public SQLType { public: SQLBit(const std::string&, unsigned long, unsigned long); ~SQLBit(); unsigned long mask() const { return mask_; } unsigned long shift() const { return shift_; } private: // No copy allowed SQLBit(const SQLBit&); SQLBit& operator=(const SQLBit&); unsigned long mask_; unsigned long shift_; // -- Overridden methods // None size_t size() const override; using SQLType::output; void output(SQLOutput&, double, bool) const override; int getKind() const override { return integerType; } std::string asString(const double* val) const override; // friend std::ostream& operator<<(std::ostream& s,const SQLBit& p) // { p.print(s); return s; } }; } // namespace eckit::sql::type #endif eckit-2.0.7/src/eckit/sql/type/SQLInt.cc0000664000175000017500000000221415161702250020070 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/type/SQLInt.h" #include "eckit/sql/SQLOutput.h" #include "eckit/utils/Translator.h" namespace eckit::sql::type { //---------------------------------------------------------------------------------------------------------------------- SQLInt::SQLInt(const std::string& name) : SQLType(name) {} SQLInt::~SQLInt() {} size_t SQLInt::size() const { return sizeof(long); } void SQLInt::output(SQLOutput& o, double d, bool missing) const { o.outputInt(d, missing); } std::string SQLInt::asString(const double* val) const { return eckit::Translator()(*val); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::type eckit-2.0.7/src/eckit/sql/type/SQLReal.cc0000664000175000017500000000221415161702250020221 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/type/SQLReal.h" #include "eckit/sql/SQLOutput.h" #include "eckit/utils/Translator.h" namespace eckit::sql::type { //---------------------------------------------------------------------------------------------------------------------- SQLReal::SQLReal(const std::string& name) : SQLType(name) {} SQLReal::~SQLReal() {} size_t SQLReal::size() const { return sizeof(double); } void SQLReal::output(SQLOutput& o, double d, bool m) const { o.outputReal(d, m); } std::string SQLReal::asString(const double* val) const { return eckit::Translator()(*val); } //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit::sql::type eckit-2.0.7/src/eckit/sql/type/SQLType.h0000664000175000017500000000504015161702250020121 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLType.h // Baudouin Raoult - ECMWF Dec 03 #ifndef eckit_sql_SQLType_H #define eckit_sql_SQLType_H #include namespace eckit::sql { //---------------------------------------------------------------------------------------------------------------------- class SQLOutput; namespace type { class SQLType { public: enum { realType = 0, integerType = 1, stringType = 2, bitmapType = 3, blobType = 4, doubleType = 5 }; SQLType(const std::string&); SQLType(const SQLType&) = delete; SQLType& operator=(const SQLType&) = delete; SQLType(SQLType&&) = delete; SQLType& operator=(SQLType&&) = delete; virtual ~SQLType(); const std::string& name() const { return name_; } virtual size_t size() const = 0; virtual void output(SQLOutput&, double, bool) const = 0; virtual void output(SQLOutput& out, const double* val, bool missing) const { output(out, *val, missing); } virtual std::string asString(const double* val) const = 0; virtual const SQLType* subType(const std::string&) const; virtual int getKind() const = 0; // Formating functions (used by SQLSimpleOutput) virtual size_t width() const; using manipulator = std::ios_base& (*)(std::ios_base&); virtual manipulator format() const; static const SQLType& lookup(const std::string&, size_t sizeDoubles = 1); static void createAlias(const std::string&, const std::string&); // n.b. takes ownership static SQLType* registerType(SQLType*); protected: virtual void print(std::ostream&) const; static bool exists(const std::string&); private: std::string name_; friend std::ostream& operator<<(std::ostream& s, const SQLType& p) { p.print(s); return s; } }; class DynamicallyCreatedTypesDestroyer { public: static SQLType* registerType(SQLType*); ~DynamicallyCreatedTypesDestroyer(); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace type } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/type/SQLInt.h0000664000175000017500000000222515161702250017734 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLInt.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLInt_H #define SQLInt_H #include "eckit/sql/type/SQLType.h" namespace eckit::sql { class SQLOutput; namespace type { class SQLInt : public SQLType { public: SQLInt(const std::string&); ~SQLInt(); private: // No copy allowed SQLInt(const SQLInt&); SQLInt& operator=(const SQLInt&); size_t size() const override; using SQLType::output; void output(SQLOutput& s, double, bool) const override; int getKind() const override { return integerType; } std::string asString(const double* val) const override; // friend std::ostream& operator<<(std::ostream& s,const SQLInt& p) // { p.print(s); return s; } }; } // namespace type } // namespace eckit::sql #endif eckit-2.0.7/src/eckit/sql/type/SQLBitfield.cc0000664000175000017500000000675115161702250021072 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/type/SQLBitfield.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLOutput.h" #include "eckit/sql/SQLTypedefs.h" #include "eckit/sql/type/SQLBit.h" #include "eckit/types/Types.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" using namespace eckit; namespace eckit::sql::type { SQLBitfield::SQLBitfield(const std::string& name, const FieldNames& fields, const Sizes& sizes) : SQLType(name), bitfieldDef_(make_pair(fields, sizes)) { int shift = 0; for (size_t i = 0; i < fields.size(); i++) { shift_[fields[i]] = shift; mask_[fields[i]] = makeMask(sizes[i]) << shift; shift += sizes[i]; } width_ = shift; } SQLBitfield::~SQLBitfield() {} unsigned long SQLBitfield::mask(const std::string& n) const { std::map::const_iterator j = mask_.find(n); if (j == mask_.end()) { throw eckit::UserError("SQLBitfield no field", n); } return (*j).second; } unsigned long SQLBitfield::shift(const std::string& n) const { std::map::const_iterator j = shift_.find(n); if (j == shift_.end()) { throw eckit::UserError("SQLBitfield no field", n); } return (*j).second; } std::string SQLBitfield::make(const std::string& name, const FieldNames& fields, const Sizes& sizes, const char* ddlName) { std::stringstream s; s << name << "["; for (size_t i = 0; i < fields.size(); ++i) { s << fields[i] << ":" << Translator()(sizes[i]) << ((i + 1 != fields.size()) ? ";" : ""); } s << "]"; std::string typeName = s.str(); if (!exists(typeName)) { // Ownership of SQLBitfield assumed by TypeRegistry SQLType::registerType(new SQLBitfield(typeName, fields, sizes)); if (ddlName) { SQLType::createAlias(typeName, ddlName); } } return typeName; } size_t SQLBitfield::width() const { return width_; } size_t SQLBitfield::size() const { return sizeof(long); } void SQLBitfield::output(SQLOutput& o, double d, bool missing) const { o.outputBitfield(d, missing); } std::string SQLBitfield::asString(const double* val) const { return eckit::Translator()(*val); } const SQLType* SQLBitfield::subType(const std::string& name) const { std::vector v; Tokenizer(".@")(name, v); if (v.size() == 1) { return this; } if (v.size() == 2 && name.find('@') != std::string::npos) { return this; } ASSERT(v.size() == 3 || v.size() == 2); // name was e.g: "status.active@body" or "status.active" std::string field = v[1]; std::string full = name; // this->name() + "." + field; if (exists(full)) { return &lookup(full); } return SQLType::registerType(new SQLBit(full, mask(field), shift(field))); } unsigned long SQLBitfield::makeMask(unsigned long size) { unsigned long mask = 0; while (size--) { mask <<= 1; mask |= 1; } return mask; } } // namespace eckit::sql::type eckit-2.0.7/src/eckit/sql/type/SQLString.cc0000664000175000017500000000277215161702250020615 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/type/SQLString.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLOutput.h" namespace eckit::sql::type { SQLString::SQLString(const std::string& name, size_t maxLen) : SQLType(name), maxLen_(maxLen) { ASSERT(maxLen_ % 8 == 0); } SQLString::~SQLString() {} size_t SQLString::size() const { return maxLen_; } void SQLString::output(SQLOutput& o, double d, bool missing) const { throw SeriousBug("We should never hit this override", Here()); } void SQLString::output(SQLOutput& o, const double* d, bool missing) const { if (missing) { o.outputString(nullptr, 0, missing); } else { const char* c(reinterpret_cast(d)); o.outputString(c, ::strnlen(c, maxLen_), missing); } } std::string SQLString::asString(const double* val) const { const char* c(reinterpret_cast(val)); return std::string(c, ::strnlen(c, maxLen_)); } SQLType::manipulator SQLString::format() const { return &std::left; } size_t SQLString::width() const { return maxLen_ + 2; } } // namespace eckit::sql::type eckit-2.0.7/src/eckit/sql/SQLStatement.h0000664000175000017500000000220415161702250020162 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File SQLStatement.h // Baudouin Raoult - ECMWF Dec 03 #ifndef SQLStatement_H #define SQLStatement_H #include "eckit/sql/expression/SQLExpressions.h" namespace eckit::sql { // Forward declarations class SQLDatabase; class SQLStatement { public: SQLStatement(); virtual ~SQLStatement(); virtual unsigned long long execute() = 0; virtual expression::Expressions output() const = 0; protected: virtual void print(std::ostream&) const; private: // No copy allowed SQLStatement(const SQLStatement&); SQLStatement& operator=(const SQLStatement&); friend std::ostream& operator<<(std::ostream& s, const SQLStatement& p) { p.print(s); return s; } }; } // namespace eckit::sql #endif eckit-2.0.7/src/CMakeLists.txt0000664000175000017500000000103715161702250016332 0ustar alastairalastair# (C) Copyright 1996- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. ### sources if( HAVE_EXPERIMENTAL ) add_subdirectory( experimental ) endif() add_subdirectory( eckit ) add_subdirectory( tools ) add_subdirectory( sandbox ) eckit-2.0.7/.git-blame-ignore-revs0000664000175000017500000000070015161702250017077 0ustar alastairalastair# The following refs can be ignored by git blame if 'git blame' is called with # --ignore-revs-file .git-blame-ignore-revs # Alternatively configure git to always ignore refs stored in this file by # calling: # git config blame.ignoreRevsFile .git-blame-ignore-revs # Or extend your git config manually with # [blame] # ignoreRevsFile = .git-blame-ignore-revs # Reformatted codebase with clang-format 2b05a8023ff3b4aef2d6047a0ab38ac38d9de026 eckit-2.0.7/eckit-import.cmake.in0000664000175000017500000000005115161702250017014 0ustar alastairalastairset( ECKIT_LIBRARIES @ECKIT_LIBRARIES@ ) eckit-2.0.7/share/0000775000175000017500000000000015161702250014104 5ustar alastairalastaireckit-2.0.7/share/eckit/0000775000175000017500000000000015161702250015203 5ustar alastairalastaireckit-2.0.7/share/eckit/geo/0000775000175000017500000000000015161702250015755 5ustar alastairalastaireckit-2.0.7/share/eckit/geo/FESOM.yaml0000664000175000017500000000431515161702250017515 0ustar alastairalastair--- grid_names: - pi_C: &pi_C type: FESOM name: pi fesom_arrangement: C shape: 5839 fesom_uid: "e548b74fa53eef5ab412c6061330f043" url_connectivity: grid/fesom/pi.fesom-conn.ek url: grid/fesom/pi_C.fesom.ek - pi_N: &pi_N type: FESOM name: pi fesom_arrangement: N shape: 3140 fesom_uid: "bdc49d97a27e389fb86decd08a185c2f" url_connectivity: grid/fesom/pi.fesom-conn.ek url: grid/fesom/pi_N.fesom.ek - CORE2_C: &CORE2_C type: FESOM name: CORE2 fesom_arrangement: C shape: 244659 fesom_uid: "9930995170a9d054187b13962a648880" url_connectivity: grid/fesom/CORE2.fesom-conn.ek url: grid/fesom/CORE2_C.fesom.ek - CORE2_N: &CORE2_N type: FESOM name: CORE2 fesom_arrangement: N shape: 126858 fesom_uid: "1d03f0efefdc1a3a8798ad53ee983140" url_connectivity: grid/fesom/CORE2.fesom-conn.ek url: grid/fesom/CORE2_N.fesom.ek - DART_C: &DART_C type: FESOM name: DART fesom_arrangement: C shape: 6262485 fesom_uid: "de015703cf0c3b339a9f0f6ec080e4f6" url_connectivity: grid/fesom/DART.fesom-conn.ek url: grid/fesom/DART_C.fesom.ek - DART_N: &DART_N type: FESOM name: DART fesom_arrangement: N shape: 3160340 fesom_uid: "35e958b1dfd9054bb86f283d75ccd7b4" url_connectivity: grid/fesom/DART.fesom-conn.ek url: grid/fesom/DART_N.fesom.ek - NG5_C: &NG5_C type: FESOM name: NG5 fesom_arrangement: C shape: 14741520 fesom_uid: "9b6c413e35230d72a4ab69242d0d0617" url_connectivity: grid/fesom/NG5.fesom-conn.ek url: grid/fesom/NG5_C.fesom.ek - NG5_N: &NG5_N type: FESOM name: NG5 fesom_arrangement: N shape: 7402886 fesom_uid: "7a33ecaebe22e7d5ca4abbd819a43240" url_connectivity: grid/fesom/NG5.fesom-conn.ek url: grid/fesom/NG5_N.fesom.ek grid_uids: - "e548b74fa53eef5ab412c6061330f043": *pi_C - "bdc49d97a27e389fb86decd08a185c2f": *pi_N - "9930995170a9d054187b13962a648880": *CORE2_C - "1d03f0efefdc1a3a8798ad53ee983140": *CORE2_N - "de015703cf0c3b339a9f0f6ec080e4f6": *DART_C - "35e958b1dfd9054bb86f283d75ccd7b4": *DART_N - "9b6c413e35230d72a4ab69242d0d0617": *NG5_C - "7a33ecaebe22e7d5ca4abbd819a43240": *NG5_N eckit-2.0.7/share/eckit/geo/area.yaml0000664000175000017500000000031515161702250017550 0ustar alastairalastair--- area_names: - globe-prime-meridian: type: bounding_box bounding_box: [90., 0., -90., 360.] - globe-antimeridian: type: bounding_box bounding_box: [90., -180., -90., 180.] eckit-2.0.7/share/eckit/geo/projection.yaml0000664000175000017500000000003215161702250021010 0ustar alastairalastair--- projection_names: [] eckit-2.0.7/share/eckit/geo/ICON-CH.yaml0000664000175000017500000000145015161702250017661 0ustar alastairalastair--- grid_names: - ICON-CH1-V1: &ICON_CH1_V1 type: ICON name: ICON-CH1 shape: [1147980] bounding_box: [50.6, -0.9, 41.9, 17.8] icon_number_of_grid_used: 1 icon_number_of_grid_in_reference: 1 icon_arrangement: C icon_uid: "17643da2574959b644d254a3cd6e2bc0" url: grid/icon-ch/icon-ch1-c.ek - ICON-CH2-V1: &ICON_CH2_V1 type: ICON name: ICON-CH2 shape: [283876] bounding_box: [50.6, -0.9, 41.9, 17.8] icon_number_of_grid_used: 2 icon_number_of_grid_in_reference: 1 icon_arrangement: C icon_uid: "bbbd5a09855499243c7a4aa4c8762920" url: grid/icon-ch/icon-ch2-c.ek - ICON-CH1: *ICON_CH1_V1 - ICON-CH2: *ICON_CH2_V1 grid_uids: - "17643da2574959b644d254a3cd6e2bc0": *ICON_CH1_V1 - "bbbd5a09855499243c7a4aa4c8762920": *ICON_CH2_V1 eckit-2.0.7/share/eckit/geo/ORCA.yaml0000664000175000017500000002535515161702250017377 0ustar alastairalastair--- grid_names: - ORCA2_F: &ORCA2_F type: ORCA name: ORCA2 orca_arrangement: F dimensions: [182, 149] orca_uid: "174487fbace54b00d959d971e88b71e7" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA2_F.atlas - ORCA2_T: &ORCA2_T type: ORCA name: ORCA2 orca_arrangement: T dimensions: [182, 149] orca_uid: "d5bde4f52ff3a9bea5629cd9ac514410" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA2_T.atlas - ORCA2_U: &ORCA2_U type: ORCA name: ORCA2 orca_arrangement: U dimensions: [182, 149] orca_uid: "857f7affa3a381e3882d38d321384e49" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA2_U.atlas - ORCA2_V: &ORCA2_V type: ORCA name: ORCA2 orca_arrangement: V dimensions: [182, 149] orca_uid: "ca637bc5dc9a54e2ea4b9750e1b79e6e" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA2_V.atlas - ORCA2_W: &ORCA2_W type: ORCA name: ORCA2 orca_arrangement: W dimensions: [182, 149] orca_uid: "edea6f71eb558dc056b5f576d5b904f7" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA2_T.atlas - ORCA1_F: &ORCA1_F type: ORCA name: ORCA1 orca_arrangement: F dimensions: [362, 292] orca_uid: "a832a12030c73928133553ec3a8d2a7e" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA1_F.atlas - ORCA1_T: &ORCA1_T type: ORCA name: ORCA1 orca_arrangement: T dimensions: [362, 292] orca_uid: "f4c91b6233fe55dec992160ec12b38df" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA1_T.atlas - ORCA1_U: &ORCA1_U type: ORCA name: ORCA1 orca_arrangement: U dimensions: [362, 292] orca_uid: "1b0f8d234753f910197c975c906b4da5" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA1_U.atlas - ORCA1_V: &ORCA1_V type: ORCA name: ORCA1 orca_arrangement: V dimensions: [362, 292] orca_uid: "c637340454795b395f982851b840943d" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA1_V.atlas - ORCA1_W: &ORCA1_W type: ORCA name: ORCA1 orca_arrangement: W dimensions: [362, 292] orca_uid: "d50061c43e83c46c3810002591ea21e1" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA1_T.atlas - eORCA1_F: &eORCA1_F type: ORCA name: eORCA1 orca_arrangement: F dimensions: [362, 332] orca_uid: "3c6d95561710c6f39b394809ff6c588c" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA1_F.atlas - eORCA1_T: &eORCA1_T type: ORCA name: eORCA1 orca_arrangement: T dimensions: [362, 332] orca_uid: "ba65665a9e68d1a8fa0352ecfcf8e496" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA1_T.atlas - eORCA1_U: &eORCA1_U type: ORCA name: eORCA1 orca_arrangement: U dimensions: [362, 332] orca_uid: "4eb1054957dcae914e219faf9a4068e3" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA1_U.atlas - eORCA1_V: &eORCA1_V type: ORCA name: eORCA1 orca_arrangement: V dimensions: [362, 332] orca_uid: "09131429766e7737c087d3a8d7073dc9" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA1_V.atlas - eORCA1_W: &eORCA1_W type: ORCA name: eORCA1 orca_arrangement: W dimensions: [362, 332] orca_uid: "5c678d8f9aa2edfbf57246d11d9c1278" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA1_T.atlas - ORCA025_F: &ORCA025_F type: ORCA name: ORCA025 orca_arrangement: F dimensions: [1442, 1021] orca_uid: "efbc280d8d4b6048797880da2605bacb" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA025_F.atlas - ORCA025_T: &ORCA025_T type: ORCA name: ORCA025 orca_arrangement: T dimensions: [1442, 1021] orca_uid: "15c961c269ac182ca226d7195f3921ba" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA025_T.atlas - ORCA025_U: &ORCA025_U type: ORCA name: ORCA025 orca_arrangement: U dimensions: [1442, 1021] orca_uid: "3f4a68bc5b54c9f867fbcc12aacc723d" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA025_U.atlas - ORCA025_V: &ORCA025_V type: ORCA name: ORCA025 orca_arrangement: V dimensions: [1442, 1021] orca_uid: "9c87699ee2026c0feee07d2a972eaccd" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA025_V.atlas - ORCA025_W: &ORCA025_W type: ORCA name: ORCA025 orca_arrangement: W dimensions: [1442, 1021] orca_uid: "74ca68f1c8524811f3d3aad99536adc2" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA025_T.atlas - eORCA025_F: &eORCA025_F type: ORCA name: eORCA025 orca_arrangement: F dimensions: [1442, 1207] orca_uid: "770e5bbb667a253d55db8a98a3b2d3a9" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA025_F.atlas - eORCA025_T: &eORCA025_T type: ORCA name: eORCA025 orca_arrangement: T dimensions: [1442, 1207] orca_uid: "983412216c9768bc794c18dc92082895" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA025_T.atlas - eORCA025_U: &eORCA025_U type: ORCA name: eORCA025 orca_arrangement: U dimensions: [1442, 1207] orca_uid: "b1b2922e9b57ee9c6eeddad218b6e4f3" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA025_U.atlas - eORCA025_V: &eORCA025_V type: ORCA name: eORCA025 orca_arrangement: V dimensions: [1442, 1207] orca_uid: "9b06bf73a8f14e927bd9b0f1f0c04f74" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA025_V.atlas - eORCA025_W: &eORCA025_W type: ORCA name: eORCA025 orca_arrangement: W dimensions: [1442, 1207] orca_uid: "4a1ba3b11b8888aefc96992b6b1cab62" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA025_T.atlas - ORCA12_F: &ORCA12_F type: ORCA name: ORCA12 orca_arrangement: F dimensions: [4322, 3059] orca_uid: "29693ad8a7af3ae3ee0f02d090f0ec7b" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA12_F.atlas - ORCA12_T: &ORCA12_T type: ORCA name: ORCA12 orca_arrangement: T dimensions: [4322, 3059] orca_uid: "b117d01170ac77bca68560ab10e559de" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA12_T.atlas - ORCA12_U: &ORCA12_U type: ORCA name: ORCA12 orca_arrangement: U dimensions: [4322, 3059] orca_uid: "fff193b92d94d03e847ff2fa62b493f4" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA12_U.atlas - ORCA12_V: &ORCA12_V type: ORCA name: ORCA12 orca_arrangement: V dimensions: [4322, 3059] orca_uid: "986e3450774b716f6e75c1987e370b10" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA12_V.atlas - ORCA12_W: &ORCA12_W type: ORCA name: ORCA12 orca_arrangement: W dimensions: [4322, 3059] orca_uid: "ccfe953619a8dd49a7f765923882a274" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/ORCA12_T.atlas - eORCA12_F: &eORCA12_F type: ORCA name: eORCA12 orca_arrangement: F dimensions: [4322, 3606] orca_uid: "25da53ed581b3931fa310840fa9aefd9" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA12_F.atlas - eORCA12_T: &eORCA12_T type: ORCA name: eORCA12 orca_arrangement: T dimensions: [4322, 3606] orca_uid: "1553b66f5885cf5f83ad4b4fdf25f460" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA12_T.atlas - eORCA12_U: &eORCA12_U type: ORCA name: eORCA12 orca_arrangement: U dimensions: [4322, 3606] orca_uid: "3e87c826643da440b4e9d9f67588a576" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA12_U.atlas - eORCA12_V: &eORCA12_V type: ORCA name: eORCA12 orca_arrangement: V dimensions: [4322, 3606] orca_uid: "cc1e3fc06a2cd18c0653e557510b8a71" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA12_V.atlas - eORCA12_W: &eORCA12_W type: ORCA name: eORCA12 orca_arrangement: W dimensions: [4322, 3606] orca_uid: "462469edbd0e0586a0cf17424cc58c89" url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA12_T.atlas grid_uids: - "174487fbace54b00d959d971e88b71e7": *ORCA2_F - "d5bde4f52ff3a9bea5629cd9ac514410": *ORCA2_T - "857f7affa3a381e3882d38d321384e49": *ORCA2_U - "ca637bc5dc9a54e2ea4b9750e1b79e6e": *ORCA2_V - "edea6f71eb558dc056b5f576d5b904f7": *ORCA2_W - "a832a12030c73928133553ec3a8d2a7e": *ORCA1_F - "f4c91b6233fe55dec992160ec12b38df": *ORCA1_T - "1b0f8d234753f910197c975c906b4da5": *ORCA1_U - "c637340454795b395f982851b840943d": *ORCA1_V - "d50061c43e83c46c3810002591ea21e1": *ORCA1_W - "3c6d95561710c6f39b394809ff6c588c": *eORCA1_F - "ba65665a9e68d1a8fa0352ecfcf8e496": *eORCA1_T - "4eb1054957dcae914e219faf9a4068e3": *eORCA1_U - "09131429766e7737c087d3a8d7073dc9": *eORCA1_V - "5c678d8f9aa2edfbf57246d11d9c1278": *eORCA1_W - "efbc280d8d4b6048797880da2605bacb": *ORCA025_F - "15c961c269ac182ca226d7195f3921ba": *ORCA025_T - "3f4a68bc5b54c9f867fbcc12aacc723d": *ORCA025_U - "9c87699ee2026c0feee07d2a972eaccd": *ORCA025_V - "74ca68f1c8524811f3d3aad99536adc2": *ORCA025_W - "770e5bbb667a253d55db8a98a3b2d3a9": *eORCA025_F - "983412216c9768bc794c18dc92082895": *eORCA025_T - "b1b2922e9b57ee9c6eeddad218b6e4f3": *eORCA025_U - "9b06bf73a8f14e927bd9b0f1f0c04f74": *eORCA025_V - "4a1ba3b11b8888aefc96992b6b1cab62": *eORCA025_W - "29693ad8a7af3ae3ee0f02d090f0ec7b": *ORCA12_F - "b117d01170ac77bca68560ab10e559de": *ORCA12_T - "fff193b92d94d03e847ff2fa62b493f4": *ORCA12_U - "986e3450774b716f6e75c1987e370b10": *ORCA12_V - "ccfe953619a8dd49a7f765923882a274": *ORCA12_W - "25da53ed581b3931fa310840fa9aefd9": *eORCA12_F - "1553b66f5885cf5f83ad4b4fdf25f460": *eORCA12_T - "3e87c826643da440b4e9d9f67588a576": *eORCA12_U - "cc1e3fc06a2cd18c0653e557510b8a71": *eORCA12_V - "462469edbd0e0586a0cf17424cc58c89": *eORCA12_W - "16076978a048410747dd7c9876677b28": type: ORCA name: eORCA1 orca_arrangement: T dimensions: [362, 332] orca_uid: "16076978a048410747dd7c9876677b28" # (uid older version) url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA1_T.atlas - "7378487847e050559b82d0792374a705": type: ORCA name: eORCA1 orca_arrangement: U dimensions: [362, 332] orca_uid: "7378487847e050559b82d0792374a705" # (uid older version) url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA1_U.atlas - "d9622b55f3120eafb3dbaf5c742bc56c": type: ORCA name: eORCA1 orca_arrangement: V dimensions: [362, 332] orca_uid: "d9622b55f3120eafb3dbaf5c742bc56c" # (uid older version) url: https://sites.ecmwf.int/repository/atlas/grids/orca/v0/eORCA1_V.atlas eckit-2.0.7/share/eckit/geo/ICON-DWD.yaml0000664000175000017500000007357415161702250020025 0ustar alastairalastair--- grid_names: - icon-grid-0001-r02b05-r: &icon_grid_0001_r02b05_r description: "Reduced radiation grid (global, R02B05). 80 km resolution." type: ICON name: icon-grid-0001-r02b05-r file: "icon_grid_0001_R02B05_R.nc" shape: [81920] icon_number_of_grid_used: 1 icon_type: hrz_radiation icon_arrangement: C icon_uid: "eee9e7881dd111b2aedbc760324270de" url: grid/icon/ICON_01_R02B05.icon.ek - icon-grid-0002-r02b06-g: &icon_grid_0002_r02b06_g description: "Global R02B06 grid. 40 km resolution." type: ICON name: icon-grid-0002-r02b06-g file: "icon_extpar_0002_R02B06_G.g2" shape: [327680] icon_number_of_grid_used: 2 icon_type: hrz_global icon_arrangement: C icon_uid: "fc9f45081dd111b2aedbc760324270de" url: grid/icon/ICON_02_R02B06.icon.ek - icon-grid-0003-r02b06-r: &icon_grid_0003_r02b06_r description: "Reduced radiation grid (global, R02B06). 40 km resolution." type: ICON name: icon-grid-0003-r02b06-r file: "icon_grid_0003_R02B06_R.nc" shape: [327680] icon_number_of_grid_used: 3 icon_type: hrz_radiation icon_arrangement: C icon_uid: "5454c1381dd211b28653972efa5f77ae" url: grid/icon/ICON_03_R02B06.icon.ek - icon-grid-0004-r02b07-g: &icon_grid_0004_r02b07_g description: "Global R02B07 grid. 20 km resolution." type: ICON name: icon-grid-0004-r02b07-g file: "icon_extpar_0004_R02B07_G.g2" shape: [1310720] icon_number_of_grid_used: 4 icon_type: hrz_global icon_arrangement: C icon_uid: "8bb13be81dd211b2bdd32f117dd15b92" url: grid/icon/ICON_04_R02B07.icon.ek - icon-grid-0005-r03b06-r: &icon_grid_0005_r03b06_r description: "Reduced radiation grid (global, R03B06)." type: ICON name: icon-grid-0005-r03b06-r file: "icon_grid_0005_R03B06_R.nc" shape: [737280] icon_number_of_grid_used: 5 icon_type: hrz_radiation icon_arrangement: C icon_uid: "dbf728c41dd211b2aa487ff9d2535f37" url: grid/icon/ICON_05_R03B06.icon.ek - icon-grid-0006-r03b07-g: &icon_grid_0006_r03b07_g description: "Global R03B07 grid." type: ICON name: icon-grid-0006-r03b07-g file: "icon_extpar_0006_R03B07_G.g2" shape: [2949120] icon_number_of_grid_used: 6 icon_type: hrz_global icon_arrangement: C icon_uid: "5a2cfeee1dd311b291c0b7d12a12800c" url: grid/icon/ICON_06_R03B07.icon.ek - icon-grid-0007-r02b04-r: &icon_grid_0007_r02b04_r description: "Reduced radiation grid (global, R02B04). 160 km resolution." type: ICON name: icon-grid-0007-r02b04-r file: "icon_grid_0007_R02B04_R.nc" shape: [20480] icon_number_of_grid_used: 7 icon_type: hrz_radiation icon_arrangement: C icon_uid: "0af76ee01dce11b28d71ada98b503cdc" url: grid/icon/ICON_07_R02B04.icon.ek - icon-grid-0008-r02b05-g: &icon_grid_0008_r02b05_g description: "Global R02B05 grid. 80 km resolution." type: ICON name: icon-grid-0008-r02b05-g file: "icon_extpar_0008_R02B05_G.g2" shape: [81920] icon_number_of_grid_used: 8 icon_type: hrz_global icon_arrangement: C icon_uid: "0df795de1dce11b28c3f4f045d5627af" url: grid/icon/ICON_08_R02B05.icon.ek - icon-grid-0009-r02b03-r: &icon_grid_0009_r02b03_r description: "Reduced radiation grid (global, R02B03). 320 km resolution." type: ICON name: icon-grid-0009-r02b03-r file: "icon_grid_0009_R02B03_R.nc" shape: [5120] icon_number_of_grid_used: 9 icon_type: hrz_radiation icon_arrangement: C icon_uid: "65dadcba1dcf11b2880c0f1645f3d1dc" url: grid/icon/ICON_09_R02B03.icon.ek - icon-grid-0010-r02b04-g: &icon_grid_0010_r02b04_g description: "Global R02B04 grid. 160 km resolution." type: ICON name: icon-grid-0010-r02b04-g file: "icon_extpar_0010_R02B04_G.g2" shape: [20480] icon_number_of_grid_used: 10 icon_type: hrz_global icon_arrangement: C icon_uid: "66a341d21dcf11b2880c0f1645f3d1dc" url: grid/icon/ICON_10_R02B04.icon.ek - icon-grid-0011-r02b03-r: &icon_grid_0011_r02b03_r description: "Reduced radiation grid (global, R02B03). 320 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0011-r02b03-r file: "icon_grid_0011_R02B03_R.nc" icon_number_of_grid_used: 11 icon_type: hrz_radiation icon_arrangement: C shape: [5120] icon_uid: "95b2202a1dce11b28af295fabaab0373" url: grid/icon/ICON_11_R02B03.icon.ek - icon-grid-0012-r02b04-g: &icon_grid_0012_r02b04_g description: "Global R02B04 grid. 160 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0012-r02b04-g file: "icon_extpar_0012_R02B04_G_20131001.g2" shape: [20480] icon_number_of_grid_used: 12 icon_type: hrz_global icon_arrangement: C icon_uid: "968657501dce11b29969d72c2fa229cf" url: grid/icon/ICON_12_R02B04.icon.ek - icon-grid-0013-r02b04-r: &icon_grid_0013_r02b04_r description: "Reduced radiation grid (global, R02B04). 160 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0013-r02b04-r file: "icon_grid_0013_R02B04_R.nc" shape: [20480] icon_number_of_grid_used: 13 icon_type: hrz_radiation icon_arrangement: C icon_uid: "af122aca1dd211b2a7f8c7bf6bc21eba" url: grid/icon/ICON_13_R02B04.icon.ek - icon-grid-0014-r02b05-g: &icon_grid_0014_r02b05_g description: "Global R02B05 grid. 80 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0014-r02b05-g file: "icon_extpar_0014_R02B05_G_20131001.g2" shape: [81920] icon_number_of_grid_used: 14 icon_type: hrz_global icon_arrangement: C icon_uid: "b203754a1dd211b284a9193e147d086f" url: grid/icon/ICON_14_R02B05.icon.ek - icon-grid-0015-r02b05-r: &icon_grid_0015_r02b05_r description: "Reduced radiation grid (global, R02B05). 80 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0015-r02b05-r file: "icon_grid_0015_R02B05_R.nc" shape: [81920] icon_number_of_grid_used: 15 icon_type: hrz_radiation icon_arrangement: C icon_uid: "5f89b5121dd311b29399f117ec0ad4d5" url: grid/icon/ICON_15_R02B05.icon.ek - icon-grid-0016-r02b06-g: &icon_grid_0016_r02b06_g description: "Global R02B06 grid. 40 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0016-r02b06-g file: "icon_extpar_0016_R02B06_G_20131001.g2" shape: [327680] icon_number_of_grid_used: 16 icon_type: hrz_global icon_arrangement: C icon_uid: "6bd687a01dd311b29399f117ec0ad4d5" url: grid/icon/ICON_16_R02B06.icon.ek - icon-grid-0017-r02b06-r: &icon_grid_0017_r02b06_r description: "Reduced radiation grid (global, R02B06). 40 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0017-r02b06-r file: "icon_grid_0017_R02B06_R.nc" shape: [327680] icon_number_of_grid_used: 17 icon_type: hrz_radiation icon_arrangement: C icon_uid: "604969921dd411b2a9d1b5e674a856dd" url: grid/icon/ICON_17_R02B06.icon.ek - icon-grid-0018-r02b07-g: &icon_grid_0018_r02b07_g description: "Global R02B07 grid. 20 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0018-r02b07-g file: "icon_extpar_0018_R02B07_G_20131001.g2" shape: [1310720] icon_number_of_grid_used: 18 icon_type: hrz_global icon_arrangement: C icon_uid: "8f2497641dd411b2a9d1b5e674a856dd" url: grid/icon/ICON_18_R02B07.icon.ek - icon-grid-0019-r03b05-r: &icon_grid_0019_r03b05_r description: "Reduced radiation grid (global, R03B05). 53 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0019-r03b05-r file: "icon_grid_0019_R03B05_R.nc" shape: [184320] icon_number_of_grid_used: 19 icon_type: hrz_radiation icon_arrangement: C icon_uid: "1c01cfa21dd611b281fc65db9ac7d8f6" url: grid/icon/ICON_19_R03B05.icon.ek - icon-grid-0020-r03b06-g: &icon_grid_0020_r03b06_g description: "Global R03B06 grid. 26 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0020-r03b06-g file: "icon_extpar_0020_R03B06_G_20131001.g2" shape: [737280] icon_number_of_grid_used: 20 icon_type: hrz_global icon_arrangement: C icon_uid: "365801d21dd611b2b98a6b5c1380b5e3" url: grid/icon/ICON_20_R03B06.icon.ek - icon-grid-0021-r03b06-r: &icon_grid_0021_r03b06_r description: "Reduced radiation grid (global, R03B06). 26 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0021-r03b06-r file: "icon_grid_0021_R03B06_R.nc" shape: [737280] icon_number_of_grid_used: 21 icon_type: hrz_radiation icon_arrangement: C icon_uid: "eb8c0fa81dd611b2837c051661b31a74" url: grid/icon/ICON_21_R03B06.icon.ek - icon-grid-0022-r03b07-g: &icon_grid_0022_r03b07_g description: "Global R03B07 grid. 13 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0022-r03b07-g file: "icon_extpar_0022_R03B07_G_20131001.g2" shape: [2949120] icon_number_of_grid_used: 22 icon_type: hrz_global icon_arrangement: C icon_uid: "eb8c0fa91dd611b2837c051661b31a74" url: grid/icon/ICON_22_R03B07.icon.ek - icon-grid-0023-r02b05-r: &icon_grid_0023_r02b05_r description: "Reduced radiation grid (global, R02B05). 80 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0023-r02b05-r file: "icon_grid_0023_R02B05_R.nc" shape: [81920] icon_number_of_grid_used: 23 icon_type: hrz_radiation icon_arrangement: C icon_uid: "9b0b038e18c411e49318776f158edd08" url: grid/icon/ICON_23_R02B05.icon.ek - icon-grid-0024-r02b06-g: &icon_grid_0024_r02b06_g description: "Global R02B06 grid. 40 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0024-r02b06-g file: "icon_extpar_0024_R02B06_G_20140731.g2" shape: [327680] icon_number_of_grid_used: 24 icon_type: hrz_global icon_arrangement: C icon_uid: "9b0b03ca18c411e49318776f158edd08" url: grid/icon/ICON_24_R02B06.icon.ek - icon-grid-0025-r03b06-r: &icon_grid_0025_r03b06_r description: "Reduced radiation grid (global, R03B06). 26 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0025-r03b06-r file: "icon_grid_0025_R03B06_R.nc" shape: [737280] icon_number_of_grid_used: 25 icon_type: hrz_radiation icon_arrangement: C icon_uid: "a27b8daa18c411e4820ab5b098c6a5c0" url: grid/icon/ICON_25_R03B06.icon.ek - icon-grid-0026-r03b07-g: &icon_grid_0026_r03b07_g description: "Global R03B07 grid. 13 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0026-r03b07-g file: "icon_extpar_0026_R03B07_G_20140731.g2" shape: [2949120] icon_number_of_grid_used: 26 icon_type: hrz_global icon_arrangement: C icon_uid: "a27b8de618c411e4820ab5b098c6a5c0" url: grid/icon/ICON_26_R03B07.icon.ek - icon-grid-0027-r03b08-n02: &icon_grid_0027_r03b08_n02 description: "Regional R03B08 grid (Europe). 6.5 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0027-r03b08-n02 file: "icon_extpar_0027_R03B08_N02_20150224.g2" shape: [659156] icon_number_of_grid_used: 27 icon_type: hrz_regional icon_arrangement: C icon_uid: "ec13b8bcb82d11e4b13f4d55411d42e6" url: grid/icon/ICON_27_R03B08.icon.ek - icon-grid-0028-r02b07-n02: &icon_grid_0028_r02b07_n02 description: "Regional R02B07 grid (Europe). 20 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0028-r02b07-n02 file: "icon_extpar_0028_R02B07_N02_20150521.g2" shape: [75948] icon_number_of_grid_used: 28 icon_type: hrz_regional icon_arrangement: C icon_uid: "982dcc6efe2e11e491280b03674e713a" url: grid/icon/ICON_28_R02B07.icon.ek - icon-grid-0029-r02b04-r: &icon_grid_0029_r02b04_r description: "Reduced radiation grid (global, R02B04). 160 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0029-r02b04-r file: "icon_grid_0029_R02B04_R.nc" shape: [20480] icon_number_of_grid_used: 29 icon_type: hrz_radiation icon_arrangement: C icon_uid: "a668ea8efe2e11e49a47e3e5371a87a7" url: grid/icon/ICON_29_R02B04.icon.ek - icon-grid-0030-r02b05-g: &icon_grid_0030_r02b05_g description: "Global R02B05 grid. 80 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0030-r02b05-g file: "icon_extpar_0030_R02B05_G_20150521.g2" shape: [81920] icon_number_of_grid_used: 30 icon_type: hrz_global icon_arrangement: C icon_uid: "a668eac0fe2e11e49a47e3e5371a87a7" url: grid/icon/ICON_30_R02B05.icon.ek - icon-grid-0031-r02b06-n02: &icon_grid_0031_r02b06_n02 description: "Regional R02B06 grid (Europe). 40 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0031-r02b06-n02 file: "icon_extpar_0031_R02B06_N02_20150521.g2" shape: [19744] icon_number_of_grid_used: 31 icon_type: hrz_regional icon_arrangement: C icon_uid: "a668eacafe2e11e49a47e3e5371a87a7" url: grid/icon/ICON_31_R02B06.icon.ek - icon-grid-0032-r03b04-r: &icon_grid_0032_r03b04_r description: "Reduced radiation grid (global, R03B04). 106 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0032-r03b04-r file: "icon_grid_0032_R03B04_R.nc" shape: [46080] icon_number_of_grid_used: 32 icon_type: hrz_radiation icon_arrangement: C icon_uid: "ad7af7d6fe2e11e49f0c8fef8cf63522" url: grid/icon/ICON_32_R03B04.icon.ek - icon-grid-0033-r03b05-g: &icon_grid_0033_r03b05_g description: "Global R03B05 grid. 53 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0033-r03b05-g file: "icon_extpar_0033_R03B05_G_20150521.g2" shape: [184320] icon_number_of_grid_used: 33 icon_type: hrz_global icon_arrangement: C icon_uid: "ad7af808fe2e11e49f0c8fef8cf63522" url: grid/icon/ICON_33_R03B05.icon.ek - icon-grid-0034-r03b06-n02: &icon_grid_0034_r03b06_n02 description: "Regional R03B06 grid (Europe). 26.5 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0034-r03b06-n02 file: "icon_extpar_0034_R03B06_N02_20150521.g2" shape: [42800] icon_number_of_grid_used: 34 icon_type: hrz_regional icon_arrangement: C icon_uid: "ad7af812fe2e11e49f0c8fef8cf63522" url: grid/icon/ICON_34_R03B06.icon.ek - icon-grid-0035-r03b05-r: &icon_grid_0035_r03b05_r description: "Reduced radiation grid (global, R03B05). 53 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0035-r03b05-r file: "icon_grid_0035_R03B05_R.nc" shape: [184320] icon_number_of_grid_used: 35 icon_type: hrz_radiation icon_arrangement: C icon_uid: "ae487ce2fe2e11e4af85e50a2a56a360" url: grid/icon/ICON_35_R03B05.icon.ek - icon-grid-0036-r03b06-g: &icon_grid_0036_r03b06_g description: "Global R03B06 grid. 26.5 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0036-r03b06-g file: "icon_extpar_0036_R03B06_G_20150521.g2" shape: [737280] icon_number_of_grid_used: 36 icon_type: hrz_global icon_arrangement: C icon_uid: "ae487d14fe2e11e4af85e50a2a56a360" url: grid/icon/ICON_36_R03B06.icon.ek - icon-grid-0037-r03b07-n02: &icon_grid_0037_r03b07_n02 description: "Regional R03B07 grid (Europe). 13 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0037-r03b07-n02 file: "icon_extpar_0037_R03B07_N02_20150521.g2" shape: [164984] icon_number_of_grid_used: 37 icon_type: hrz_regional icon_arrangement: C icon_uid: "ae487d28fe2e11e4af85e50a2a56a360" url: grid/icon/ICON_37_R03B07.icon.ek - icon-grid-0038-r02b06-r: &icon_grid_0038_r02b06_r description: "Reduced radiation grid (global, R02B06). 40 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0038-r02b06-r file: "icon_grid_0038_R02B06_R.nc" shape: [327680] icon_number_of_grid_used: 38 icon_type: hrz_radiation icon_arrangement: C icon_uid: "49c34088248d11e58105c12d5e3b00a4" url: grid/icon/ICON_38_R02B06.icon.ek - icon-grid-0039-r02b07-g: &icon_grid_0039_r02b07_g description: "Global R02B07 grid. 20.0 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0039-r02b07-g file: "icon_extpar_0039_R02B07_G_20150521.g2" shape: [1310720] icon_number_of_grid_used: 39 icon_type: hrz_global icon_arrangement: C icon_uid: "49c340e2248d11e58105c12d5e3b00a4" url: grid/icon/ICON_39_R02B07.icon.ek - icon-grid-0040-r02b08-n02: &icon_grid_0040_r02b08_n02 description: "Regional R02B08 grid (Europe). 10 km resolution. Grid rotated by 36 degrees around z-axis." type: ICON name: icon-grid-0040-r02b08-n02 file: "icon_extpar_0040_R02B08_N02_20150521.g2" shape: [293084] icon_number_of_grid_used: 40 icon_type: hrz_regional icon_arrangement: C icon_uid: "49c340f6248d11e58105c12d5e3b00a4" url: grid/icon/ICON_40_R02B08.icon.ek - icon-grid-0041-r02b09-lr: &icon_grid_0041_r02b09_lr description: "Radiation grid (Germany): COSMO-DE area with 5 km resolution, generated by G. Zaengl, 2018-02-01. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0041-r02b09-lr file: "icon_grid_0041_R02B09_LR.nc" shape: [63380] icon_number_of_grid_used: 41 icon_type: hrz_radiation icon_arrangement: C icon_uid: "0acedc87c1db8404c23ec6ac675a27c0" url: grid/icon/ICON_41_R02B09.icon.ek - icon-grid-0042-r02b10-l: &icon_grid_0042_r02b10_l description: "Limited area grid (Germany): COSMO-DE area with 2.5 km resolution, generated by G. Zaengl, 2018-02-01. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0042-r02b10-l file: "icon_extpar_0042_R02B10_L_20180209_tiles.g2" shape: [253520] icon_number_of_grid_used: 42 icon_type: hrz_regional icon_arrangement: C icon_uid: "d378d127082af71232e62a2093c92ba0" url: grid/icon/ICON_42_R02B10.icon.ek - icon-grid-0043-r19b06-lr: &icon_grid_0043_r19b06_lr description: "Radiation grid (Germany): COSMO-D2 area with 4 km resolution, generated by G. Zaengl, 2018-02-01. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0043-r19b06-lr file: "icon_grid_0043_R19B06_LR.nc" shape: [134541] icon_number_of_grid_used: 43 icon_type: hrz_radiation icon_arrangement: C icon_uid: "ac1190a0a846e31092a72a2bf62329c0" url: grid/icon/ICON_43_R19B06.icon.ek - icon-grid-0044-r19b07-l: &icon_grid_0044_r19b07_l description: "Limited area grid (Germany): COSMO-D2 area with 2 km resolution, generated by G. Zaengl, 2018-02-01. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0044-r19b07-l file: "icon_extpar_0044_R19B07_L_20180209_tiles.g2" shape: [538164] icon_number_of_grid_used: 44 icon_type: hrz_regional icon_arrangement: C icon_uid: "66f27a809bfa6f1056df19bf03052ce0" url: grid/icon/ICON_44_R19B07.icon.ek - icon-grid-0045-r19b08-ln02: &icon_grid_0045_r19b08_ln02 description: "Limited area grid (COSMO-D2 nest, Germany) with 1 km resolution, generated by G. Zaengl, 2018-02-01. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0045-r19b08-ln02 file: "icon_extpar_0045_R19B08_LN02_20180209_tiles.g2" shape: [716980] icon_number_of_grid_used: 45 icon_type: hrz_regional icon_arrangement: C icon_uid: "b258b3391435078ecc92d51b8a3f2ac0" url: grid/icon/ICON_45_R19B08.icon.ek - icon-grid-0046-r19b06-lr: &icon_grid_0046_r19b06_lr description: "Radiation grid (Germany): Enhanced COSMO-D2 area with 4 km resolution, generated by Ch. Koziar, 2020-05-27. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0046-r19b06-lr file: "icon_grid_0046_R19B06_LR.nc" shape: [135510] icon_number_of_grid_used: 46 icon_type: hrz_radiation icon_arrangement: C icon_uid: "626ae68aa5831aac904c6189b37a29a0" url: grid/icon/ICON_46_R19B06.icon.ek - icon-grid-0047-r19b07-l: &icon_grid_0047_r19b07_l description: "Limited area grid (Germany): Enhanced COSMO-D2 area with 2 km resolution, generated by Ch. Koziar, 2020-05-27. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0047-r19b07-l file: "icon_extpar_0047_R19B07_L_20200527_tiles.g2" shape: [542040] icon_number_of_grid_used: 47 icon_type: hrz_regional icon_arrangement: C icon_uid: "c6b12daa91ad64045b26c1b6452a2a20" url: grid/icon/ICON_47_R19B07.icon.ek - icon-grid-0048-r03b07-lr: &icon_grid_0048_r03b07_lr description: "Radiation grid (Europe): Pollen forecast area with 13 km resolution, generated by J. Foerstner, 2020-11-12. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0048-r03b07-lr file: "icon_grid_0048_R03B07_LR.nc" shape: [63057] icon_number_of_grid_used: 48 icon_type: hrz_radiation icon_arrangement: C icon_uid: "1df96f04856f9d265fc94d29354c27a0" url: grid/icon/ICON_48_R03B07.icon.ek - icon-grid-0049-r03b08-l: &icon_grid_0049_r03b08_l description: "Limited area grid (Europe): Pollen forecast area with 6.5 km resolution, generated by J. Foerstner, 2020-11-12. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0049-r03b08-l file: "icon_extpar_0049_R03B08_L_20201112_tiles.g2" shape: [252228] icon_number_of_grid_used: 49 icon_type: hrz_regional icon_arrangement: C icon_uid: "027b27853ea2c4b96436708a777b2bc0" url: grid/icon/ICON_49_R03B08.icon.ek - icon-grid-0050-r02b07-n02: &icon_grid_0050_r02b07_n02 description: "Regional R02B07 grid (Europe, North Atlantic and North Africa), generated by J. Foerstner, 2021-10-26. Bigger nest for prognostic aerosol forecasts with ICON-ART. 20 km resolution. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0050-r02b07-n02 file: "icon_extpar_0050_R02B07_N02_20211026_tiles.g2" shape: [254588] icon_number_of_grid_used: 50 icon_type: hrz_regional icon_arrangement: C icon_uid: "5eb8f1385d72f207a6d7211bb8a92ee0" url: grid/icon/ICON_50_R02B07.icon.ek - icon-grid-0051-r03b07-n02: &icon_grid_0051_r03b07_n02 description: "Regional R03B07 grid (Europe, North Atlantic and North Africa), generated by J. Foerstner, 2022-02-28. Bigger nest for prognostic aerosol forecasts with ICON-ART. 13 km resolution. Generated with DWD ICON Tools grid generator (similar to old grid generator grids rotated by 36 degrees)." type: ICON name: icon-grid-0051-r03b07-n02 file: "icon_extpar_0051_R03B07_N02_20220301_tiles.g2" shape: [571088] icon_number_of_grid_used: 51 icon_type: hrz_regional icon_arrangement: C icon_uid: "117ff92c2607ab02bc28333ae4a92c20" url: grid/icon/ICON_51_R03B07.icon.ek - icon-grid-0052-r03b07-lr: &icon_grid_0052_r03b07_lr description: "Radiation grid (Europe): Passive tracer forecast area (ITMS) with 13 km resolution, generated by B. Mamtimin, 2022-06-24. Generated with DWD ICON Tools grid generator." type: ICON name: icon-grid-0052-r03b07-lr file: "icon_grid_0052_R03B07_LR.nc" shape: [123718] icon_number_of_grid_used: 52 icon_type: hrz_radiation icon_arrangement: C icon_uid: "bcaee9844be64faa724a751445d02820" url: grid/icon/ICON_52_R03B07.icon.ek - icon-grid-0053-r03b08-l: &icon_grid_0053_r03b08_l description: "Limited area grid (Europe): Passive tracer forecast area (ITMS) with 6.5 km resolution, generated by B. Mamtimin, 2022-06-24. Generated with DWD ICON Tools grid generator." type: ICON name: icon-grid-0053-r03b08-l file: "icon_extpar_0053_R03B08_L_20231113_tiles.g2" shape: [494872] icon_number_of_grid_used: 53 icon_type: hrz_regional icon_arrangement: C icon_uid: "262ffaffe4c2b5b5ff4094b0015e2b40" url: grid/icon/ICON_53_R03B08.icon.ek - icon-grid-0054-r02b04-g: &icon_grid_0054_r02b04_g description: "Global R02B04 grid. 160 km resolution. Use case: ICON training idealized JW test. Generated by D. Reinert 2023-03-10 with DWD ICON Tools grid generator." type: ICON name: icon-grid-0054-r02b04-g file: "icon_grid_0054_R02B04_G.nc" shape: [20480] icon_number_of_grid_used: 54 icon_type: hrz_global icon_arrangement: C icon_uid: "87397f993e6f913d00fc361367e725a0" url: grid/icon/ICON_54_R02B04.icon.ek - icon-grid-0055-r02b05-n: &icon_grid_0055_r02b05_n description: "Regional R02B05 grid. 80 km resolution. Use case: ICON training idealized JW test. Generated by D. Reinert 2023-03-10 with DWD ICON Tools grid generator." type: ICON name: icon-grid-0055-r02b05-n file: "icon_grid_0055_R02B05_N.nc" shape: [2656] icon_number_of_grid_used: 55 icon_type: hrz_regional icon_arrangement: C icon_uid: "e234e01a8556e9a84bcb42361d2f24e0" url: grid/icon/ICON_55_R02B05.icon.ek - icon-grid-0056-r02b05-n: &icon_grid_0056_r02b05_n description: "Regional R02B05 grid. 80 km resolution. Use case: ICON training idealized JW test. Generated by D. Reinert 2023-03-10 with DWD ICON Tools grid generator." type: ICON name: icon-grid-0056-r02b05-n file: "icon_grid_0056_R02B05_N.nc" shape: [2628] icon_number_of_grid_used: 56 icon_type: hrz_regional icon_arrangement: C icon_uid: "a280b295c78e22bb7e4ac9bc1ac92540" url: grid/icon/ICON_56_R02B05.icon.ek grid_uids: - "eee9e7881dd111b2aedbc760324270de": *icon_grid_0001_r02b05_r - "fc9f45081dd111b2aedbc760324270de": *icon_grid_0002_r02b06_g - "5454c1381dd211b28653972efa5f77ae": *icon_grid_0003_r02b06_r - "8bb13be81dd211b2bdd32f117dd15b92": *icon_grid_0004_r02b07_g - "dbf728c41dd211b2aa487ff9d2535f37": *icon_grid_0005_r03b06_r - "5a2cfeee1dd311b291c0b7d12a12800c": *icon_grid_0006_r03b07_g - "0af76ee01dce11b28d71ada98b503cdc": *icon_grid_0007_r02b04_r - "0df795de1dce11b28c3f4f045d5627af": *icon_grid_0008_r02b05_g - "65dadcba1dcf11b2880c0f1645f3d1dc": *icon_grid_0009_r02b03_r - "66a341d21dcf11b2880c0f1645f3d1dc": *icon_grid_0010_r02b04_g - "95b2202a1dce11b28af295fabaab0373": *icon_grid_0011_r02b03_r - "968657501dce11b29969d72c2fa229cf": *icon_grid_0012_r02b04_g - "af122aca1dd211b2a7f8c7bf6bc21eba": *icon_grid_0013_r02b04_r - "b203754a1dd211b284a9193e147d086f": *icon_grid_0014_r02b05_g - "5f89b5121dd311b29399f117ec0ad4d5": *icon_grid_0015_r02b05_r - "6bd687a01dd311b29399f117ec0ad4d5": *icon_grid_0016_r02b06_g - "604969921dd411b2a9d1b5e674a856dd": *icon_grid_0017_r02b06_r - "8f2497641dd411b2a9d1b5e674a856dd": *icon_grid_0018_r02b07_g - "1c01cfa21dd611b281fc65db9ac7d8f6": *icon_grid_0019_r03b05_r - "365801d21dd611b2b98a6b5c1380b5e3": *icon_grid_0020_r03b06_g - "eb8c0fa81dd611b2837c051661b31a74": *icon_grid_0021_r03b06_r - "eb8c0fa91dd611b2837c051661b31a74": *icon_grid_0022_r03b07_g - "9b0b038e18c411e49318776f158edd08": *icon_grid_0023_r02b05_r - "9b0b03ca18c411e49318776f158edd08": *icon_grid_0024_r02b06_g - "a27b8daa18c411e4820ab5b098c6a5c0": *icon_grid_0025_r03b06_r - "a27b8de618c411e4820ab5b098c6a5c0": *icon_grid_0026_r03b07_g - "ec13b8bcb82d11e4b13f4d55411d42e6": *icon_grid_0027_r03b08_n02 - "982dcc6efe2e11e491280b03674e713a": *icon_grid_0028_r02b07_n02 - "a668ea8efe2e11e49a47e3e5371a87a7": *icon_grid_0029_r02b04_r - "a668eac0fe2e11e49a47e3e5371a87a7": *icon_grid_0030_r02b05_g - "a668eacafe2e11e49a47e3e5371a87a7": *icon_grid_0031_r02b06_n02 - "ad7af7d6fe2e11e49f0c8fef8cf63522": *icon_grid_0032_r03b04_r - "ad7af808fe2e11e49f0c8fef8cf63522": *icon_grid_0033_r03b05_g - "ad7af812fe2e11e49f0c8fef8cf63522": *icon_grid_0034_r03b06_n02 - "ae487ce2fe2e11e4af85e50a2a56a360": *icon_grid_0035_r03b05_r - "ae487d14fe2e11e4af85e50a2a56a360": *icon_grid_0036_r03b06_g - "ae487d28fe2e11e4af85e50a2a56a360": *icon_grid_0037_r03b07_n02 - "49c34088248d11e58105c12d5e3b00a4": *icon_grid_0038_r02b06_r - "49c340e2248d11e58105c12d5e3b00a4": *icon_grid_0039_r02b07_g - "49c340f6248d11e58105c12d5e3b00a4": *icon_grid_0040_r02b08_n02 - "0acedc87c1db8404c23ec6ac675a27c0": *icon_grid_0041_r02b09_lr - "d378d127082af71232e62a2093c92ba0": *icon_grid_0042_r02b10_l - "ac1190a0a846e31092a72a2bf62329c0": *icon_grid_0043_r19b06_lr - "66f27a809bfa6f1056df19bf03052ce0": *icon_grid_0044_r19b07_l - "b258b3391435078ecc92d51b8a3f2ac0": *icon_grid_0045_r19b08_ln02 - "626ae68aa5831aac904c6189b37a29a0": *icon_grid_0046_r19b06_lr - "c6b12daa91ad64045b26c1b6452a2a20": *icon_grid_0047_r19b07_l - "1df96f04856f9d265fc94d29354c27a0": *icon_grid_0048_r03b07_lr - "027b27853ea2c4b96436708a777b2bc0": *icon_grid_0049_r03b08_l - "5eb8f1385d72f207a6d7211bb8a92ee0": *icon_grid_0050_r02b07_n02 - "117ff92c2607ab02bc28333ae4a92c20": *icon_grid_0051_r03b07_n02 - "bcaee9844be64faa724a751445d02820": *icon_grid_0052_r03b07_lr - "262ffaffe4c2b5b5ff4094b0015e2b40": *icon_grid_0053_r03b08_l - "87397f993e6f913d00fc361367e725a0": *icon_grid_0054_r02b04_g - "e234e01a8556e9a84bcb42361d2f24e0": *icon_grid_0055_r02b05_n - "a280b295c78e22bb7e4ac9bc1ac92540": *icon_grid_0056_r02b05_n eckit-2.0.7/share/eckit/geo/ICON-MPIM.yaml0000664000175000017500000001024015161702250020126 0ustar alastairalastair--- grid_names: - icon-grid-0016-r02b09-o: &icon_grid_0016_r02b09_o description: "Ocean global R02B09 grid. 5 km resolution. Grid rotated by 37 degrees around z-axis and symmetrical to the equator. Grid includes ocean cells only." type: ICON name: icon-grid-0016-r02b09-o file: "icon_grid_0016_R02B09_O.nc" shape: [14886338] icon_number_of_grid_used: 16 icon_type: hrz_global_ocean_only icon_arrangement: C icon_uid: "375cb0cc637e11e89d6f8f41a9b9ff4b" url: grid/icon-mpim/ICON_O_16.icon.ek - icon-grid-0020-r02b05-o: &icon_grid_0020_r02b05_o description: "Ocean global R02B05 grid. 80 km resolution. Grid rotated by 37 degrees around z-axis and symmetrical to the equator. Grid includes ocean cells only." type: ICON name: icon-grid-0020-r02b05-o file: "icon_grid_0020_R02B05_O.nc" shape: [59261] icon_number_of_grid_used: 20 icon_type: hrz_global_ocean_only icon_arrangement: C icon_uid: "e96d69e2ab5811e89e299566f2744a3c" url: grid/icon-mpim/ICON_O_20.icon.ek - icon-grid-0024-r02b07-o: &icon_grid_0024_r02b07_o description: "Ocean global R02B07 grid. 20 km resolution. Grid rotated by 37 degrees around z-axis and symmetrical to the equator. Grid includes ocean cells only." type: ICON name: icon-grid-0024-r02b07-o file: "icon_grid_0024_R02B07_O.nc" shape: [933398] icon_number_of_grid_used: 24 icon_type: hrz_global_ocean_only icon_arrangement: C icon_uid: "28f3a5e2de9c11e88555cbf786e364f5" url: grid/icon-mpim/ICON_O_24.icon.ek - icon-grid-0036-r02b04-o: &icon_grid_0036_r02b04_o description: "Ocean global R02B04 grid. 160 km resolution. Grid rotated by 37 degrees around z-axis and symmetrical to the equator. Grid includes ocean cells only." type: ICON name: icon-grid-0036-r02b04-o file: "icon_grid_0036_R02B04_O.nc" shape: [15105] icon_number_of_grid_used: 36 icon_type: hrz_global_ocean_only icon_arrangement: C icon_uid: "5bd948e8ac1a11eaa6b1d317264fdca9" url: grid/icon-mpim/ICON_O_36.icon.ek - icon-grid-0038-r02b11-o: &icon_grid_0038_r02b11_o description: "Ocean global R02B11 grid. 1.2 km resolution. Grid rotated by 37 degrees around z-axis and symmetrical to the equator. GEBCO 2023 bathymetry. Grid includes ocean cells only." type: ICON name: icon-grid-0038-r02b11-o file: "icon_grid_0038_R02B11_O.nc" shape: [237316813] icon_number_of_grid_used: 38 icon_type: hrz_global_ocean_only icon_arrangement: C icon_uid: "9b323b62670411eb85afdd9149101e1a" url: grid/icon-mpim/ICON_O_38.icon.ek - icon-grid-0040-r02b10-o: &icon_grid_0040_r02b10_o description: "Ocean global R02B10 grid. 2.5 km resolution. Grid rotated by 37 degrees around z-axis and symmetrical to the equator. GEBCO 2023 bathymetry. Grid includes ocean cells only." type: ICON name: icon-grid-0040-r02b10-o file: "icon_grid_0040_R02B10_O.nc" shape: [59359799] icon_number_of_grid_used: 40 icon_type: hrz_global_ocean_only icon_arrangement: C icon_uid: "99c03a3c657811eb8e42f565c1ad0089" url: grid/icon-mpim/ICON_O_40.icon.ek - icon-grid-0046-r02b06-o: &icon_grid_0046_r02b06_o description: "Ocean global R02B06 grid. 40 km resolution. Grid rotated by 37 degrees around z-axis and symmetrical to the equator. Improved sill-depths in the North Atlantic (Denmark Strait, Faroer-Bank-Channel, Florida Strait, Gibraltar) Grid includes ocean cells only." type: ICON name: icon-grid-0046-r02b06-o file: "icon_grid_0046_R02B06_O.nc" shape: [235403] icon_number_of_grid_used: 46 icon_type: hrz_global_ocean_only icon_arrangement: C icon_uid: "f4ed57f6b2ea11e9ae92c52a3fa37d96" url: grid/icon-mpim/ICON_O_46.icon.ek grid_uids: - "375cb0cc637e11e89d6f8f41a9b9ff4b": *icon_grid_0016_r02b09_o - "e96d69e2ab5811e89e299566f2744a3c": *icon_grid_0020_r02b05_o - "28f3a5e2de9c11e88555cbf786e364f5": *icon_grid_0024_r02b07_o - "5bd948e8ac1a11eaa6b1d317264fdca9": *icon_grid_0036_r02b04_o - "9b323b62670411eb85afdd9149101e1a": *icon_grid_0038_r02b11_o - "99c03a3c657811eb8e42f565c1ad0089": *icon_grid_0040_r02b10_o - "f4ed57f6b2ea11e9ae92c52a3fa37d96": *icon_grid_0046_r02b06_o eckit-2.0.7/share/eckit/geo/grid.yaml0000664000175000017500000000106115161702250017564 0ustar alastairalastair--- grid_names: - LAEA-EFAS-5km: type: lambert_azimuthal_equal_area proj: +proj=laea +lat_0=52 +lon_0=10 +ellps=GRS80 +units=m +no_defs figure: grs80 first_lonlat: [-35.034024, 66.982143] grid: [5000., 5000.] shape: [1000, 950] - SMUFF-OPERA-2km: type: lambert_azimuthal_equal_area proj: +proj=laea +lat_0=55 +lon_0=10 +ellps=WGS84 +units=m +no_defs figure: wgs84 first_lonlat: [-39.535385563614, 67.035186732680] grid: [2000., 2000.] shape: [1900, 2200] eckit-2.0.7/share/eckit/geo/shapefile.yaml0000664000175000017500000002426615161702250020613 0ustar alastairalastair--- url_prefix: https://gisco-services.ec.europa.eu/distribution/v2/ area_libraries: coas-rg-01m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/coas/shp/COAS_RG_01M_2016_4326.shp.zip coas-rg-03m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/coas/shp/COAS_RG_03M_2016_4326.shp.zip coas-rg-10m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/coas/shp/COAS_RG_10M_2016_4326.shp.zip coas-rg-20m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/coas/shp/COAS_RG_20M_2016_4326.shp.zip coas-rg-60m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/coas/shp/COAS_RG_60M_2016_4326.shp.zip nuts-rg-01m-2003: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_01M_2003_4326.shp.zip nuts-rg-01m-2006: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_01M_2006_4326.shp.zip nuts-rg-01m-2010: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_01M_2010_4326.shp.zip nuts-rg-01m-2013: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_01M_2013_4326.shp.zip nuts-rg-01m-2016: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_01M_2016_4326.shp.zip nuts-rg-01m-2021: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_01M_2021_4326.shp.zip nuts-rg-01m-2024: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_01M_2024_4326.shp.zip nuts-rg-03m-2003: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_03M_2003_4326.shp.zip nuts-rg-03m-2006: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_03M_2006_4326.shp.zip nuts-rg-03m-2010: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_03M_2010_4326.shp.zip nuts-rg-03m-2013: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_03M_2013_4326.shp.zip nuts-rg-03m-2016: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_03M_2016_4326.shp.zip nuts-rg-03m-2021: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_03M_2021_4326.shp.zip nuts-rg-03m-2024: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_03M_2024_4326.shp.zip nuts-rg-10m-2003: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_10M_2003_4326.shp.zip nuts-rg-10m-2006: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_10M_2006_4326.shp.zip nuts-rg-10m-2010: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_10M_2010_4326.shp.zip nuts-rg-10m-2013: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_10M_2013_4326.shp.zip nuts-rg-10m-2016: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_10M_2016_4326.shp.zip nuts-rg-10m-2021: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_10M_2021_4326.shp.zip nuts-rg-10m-2024: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_10M_2024_4326.shp.zip nuts-rg-20m-2003: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_20M_2003_4326.shp.zip nuts-rg-20m-2006: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_20M_2006_4326.shp.zip nuts-rg-20m-2010: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_20M_2010_4326.shp.zip nuts-rg-20m-2013: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_20M_2013_4326.shp.zip nuts-rg-20m-2016: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_20M_2016_4326.shp.zip nuts-rg-20m-2021: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_20M_2021_4326.shp.zip nuts-rg-20m-2024: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_20M_2024_4326.shp.zip nuts-rg-60m-2006: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_60M_2006_4326.shp.zip nuts-rg-60m-2010: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_60M_2010_4326.shp.zip nuts-rg-60m-2013: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_60M_2013_4326.shp.zip nuts-rg-60m-2016: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_60M_2016_4326.shp.zip nuts-rg-60m-2021: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_60M_2021_4326.shp.zip nuts-rg-60m-2024: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/shp/NUTS_RG_60M_2024_4326.shp.zip ref-nuts-2003-01m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2003-01m.shp.zip ref-nuts-2003-03m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2003-03m.shp.zip ref-nuts-2003-10m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2003-10m.shp.zip ref-nuts-2003-20m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2003-20m.shp.zip ref-nuts-2006-01m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2006-01m.shp.zip ref-nuts-2006-03m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2006-03m.shp.zip ref-nuts-2006-10m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2006-10m.shp.zip ref-nuts-2006-20m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2006-20m.shp.zip ref-nuts-2006-60m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2006-60m.shp.zip ref-nuts-2010-01m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2010-01m.shp.zip ref-nuts-2010-03m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2010-03m.shp.zip ref-nuts-2010-10m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2010-10m.shp.zip ref-nuts-2010-20m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2010-20m.shp.zip ref-nuts-2010-60m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2010-60m.shp.zip ref-nuts-2013-01m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2013-01m.shp.zip ref-nuts-2013-03m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2013-03m.shp.zip ref-nuts-2013-10m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2013-10m.shp.zip ref-nuts-2013-20m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2013-20m.shp.zip ref-nuts-2013-60m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2013-60m.shp.zip ref-nuts-2016-01m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2016-01m.shp.zip ref-nuts-2016-03m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2016-03m.shp.zip ref-nuts-2016-10m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2016-10m.shp.zip ref-nuts-2016-20m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2016-20m.shp.zip ref-nuts-2016-60m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2016-60m.shp.zip ref-nuts-2021-01m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2021-01m.shp.zip ref-nuts-2021-03m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2021-03m.shp.zip ref-nuts-2021-10m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2021-10m.shp.zip ref-nuts-2021-20m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2021-20m.shp.zip ref-nuts-2021-60m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2021-60m.shp.zip ref-nuts-2024-01m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2024-01m.shp.zip ref-nuts-2024-03m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2024-03m.shp.zip ref-nuts-2024-10m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2024-10m.shp.zip ref-nuts-2024-20m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2024-20m.shp.zip ref-nuts-2024-60m: type: shapefile url: https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/ref-nuts-2024-60m.shp.zip eckit-2.0.7/share/eckit/geo/CMakeLists.txt0000664000175000017500000000104115161702250020511 0ustar alastairalastairset(_files) list(APPEND _files "area.yaml" "grid.yaml") if(eckit_HAVE_GEO_AREA_SHAPEFILE) list(APPEND _files "shapefile.yaml") endif() if(ECKIT_GEO_CODEC_GRIDS) list(APPEND _files "FESOM.yaml" "ICON-CH.yaml" "ICON-DWD.yaml" "ICON-MPIM.yaml" "ORCA.yaml") endif() set(_destination "share/eckit/geo") install(FILES ${_files} DESTINATION ${_destination} PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ) foreach(_file ${_files}) configure_file(${_file} "${CMAKE_BINARY_DIR}/${_destination}/${_file}" COPYONLY) endforeach() eckit-2.0.7/share/eckit/CMakeLists.txt0000664000175000017500000000002615161702250017741 0ustar alastairalastairadd_subdirectory(geo) eckit-2.0.7/share/CMakeLists.txt0000664000175000017500000000003015161702250016635 0ustar alastairalastairadd_subdirectory(eckit) eckit-2.0.7/python/0000775000175000017500000000000015161702250014323 5ustar alastairalastaireckit-2.0.7/python/eckitlib/0000775000175000017500000000000015161702250016111 5ustar alastairalastaireckit-2.0.7/python/eckitlib/setup.cfg0000664000175000017500000000021415161702250017727 0ustar alastairalastair[metadata] description = "eckitlib" long_description = file: README.md long_description_content_type = text/markdown author = file: AUTHORS eckit-2.0.7/python/eckitlib/buildconfig0000664000175000017500000000155115161702250020323 0ustar alastairalastair# (C) Copyright 2024- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. # to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh # NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation # TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent NAME="eckit" CMAKE_PARAMS="-DENABLE_MPI=0 -DENABLE_ECKIT_GEO=1 -DENABLE_BUILD_TOOLS=OFF -DENABLE_AEC=0 -DENABLE_EIGEN=0 -DENABLE_LZ4=1" PYPROJECT_DIR="python/eckitlib" DEPENDENCIES='[]' eckit-2.0.7/python/eckitlib/pre-compile.sh0000775000175000017500000000264215161702250020670 0ustar alastairalastair#!/bin/bash # the procedure for adding a new ext dependency to be bundled in here: # - add git checkout, compile, etc # - ensure the version ends up in python_wrapper/src/versions.txt # - ensure the licence ends up in python_wrapper/src/copying/, and fname is referenced in copying/list.json # - ensure the .so ends up in target/lib64/ with the expected libname # - validate that the resulting wheel contains all the above # additionally, make sure this script is aligned with /buildscripts/compile.sh and /buildscripts/wheel-linux.sh, # in particular when it comes to install targets and package data, etc # note also that for macos, we assume that the agent has the libraries already installed, as we can't run this in docker # we thus only prepare the license files set -euo pipefail mkdir -p python/eckitlib/src/copying mkdir -p /tmp/eckit/target/eckit/lib64/ if [ "$(uname)" != "Darwin" ] ; then echo "no deps installation for platform $(uname)" # echo "installing deps for platform $(uname)" ## lz4 # git clone https://github.com/lz4/lz4 /src/lz4 && cd /src/lz4 # make -j10 && make install DESTDIR=/tmp/lz4 # cd - else echo "no deps installation for platform $(uname)" fi wget https://raw.githubusercontent.com/lz4/lz4/dev/LICENSE -O python/eckitlib/src/copying/liblz4.txt echo '{"liblz4": {"path": "copying/liblz4.txt", "home": "https://github.com/lz4/lz4"}}' > python/eckitlib/src/copying/list.json eckit-2.0.7/python/eckitlib/post-build.sh0000775000175000017500000000125015161702250020530 0ustar alastairalastair#!/bin/bash set -euo pipefail # NOTE in case of problems like we had with eccodes, replace with noop here if [ "$(uname)" != "Darwin" ] ; then rm -rf /tmp/eckit/auditwheel auditwheel repair -w /tmp/eckit/auditwheel /tmp/eckit/build/wheel/*whl cd /tmp/eckit/auditwheel F=$(ls *whl) unzip $F patchelf --add-rpath '$ORIGIN' eckitlib.libs/* rm $F zip -r $F ./* rm /tmp/eckit/build/wheel/* mv /tmp/eckit/auditwheel/$F /tmp/eckit/build/wheel cd - fi # NOTE on macos we delocate with impunity, because the findlibs recursive depload # is disabled anyway if [ "$(uname)" = "Darwin" ] ; then delocate-wheel /tmp/eckit/build/wheel/*whl fi eckit-2.0.7/python/eckitlib/setup.py0000664000175000017500000000006215161702250017621 0ustar alastairalastairfrom setup_utils import plain_setup plain_setup() eckit-2.0.7/python/eckit/0000775000175000017500000000000015161702250015422 5ustar alastairalastaireckit-2.0.7/python/eckit/tests/0000775000175000017500000000000015161702250016564 5ustar alastairalastaireckit-2.0.7/python/eckit/tests/test_grid.py0000664000175000017500000000141615161702250021124 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import pytest SPECS = [ (dict(grid="1/1"), (181, 360)), (dict(grid="H2", ordering="nested"), (12 * 2 * 2,)), (dict(grid=[2, 2]), (91, 180)), ("{grid: 3/3}", (61, 120)), (dict(grid="o2"), (sum([20, 24, 24, 20]),)), ] @pytest.mark.parametrize( "spec, shape", SPECS) def test_grid(spec, shape): from eckit.geo import Grid grid = Grid(spec) assert shape == grid.shape eckit-2.0.7/python/eckit/tests/test_area.py0000664000175000017500000000136415161702250021111 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import pytest SPECS = [ (dict(north=90, west=0, south=-90, east=360), dict(area=[90, 0, -90, 360])), (dict(north=10, west=360*99+1, south=0, east=360*42+10), dict(area=[10, 1, 0, 10])), ] @pytest.mark.parametrize( "_spec, _expected", SPECS) def test_area(_spec, _expected): from eckit.geo import Area area = Area(_spec) assert area.spec == _expected eckit-2.0.7/python/eckit/README.rst0000664000175000017500000000130615161702250017111 0ustar alastairalastaireckit-python ============ Requires a recent version of Cython_ and Python 3:: ECKIT_SOURCE_DIR= ECKIT_BUILD_DIR= python3 setup.py build_ext -i This builds the ``eckit`` extension module in the ``build/``. The built module still requires an externally installed eckit at runtime. .. _Cython: https://cython.org/ Alternatively, one can build using the pypi-published eckit wheel -- to do so, run `PUBLISH_TO=nowhere ./build_chain.sh`. Note you need the `uv` tool installed, which then creates a local `venv` and installs cython, setup tools, eckit, etc. To build using devel version of eckit, additionally set `PIP_OVERRIDE` envvar to eg `'eckitlib==1.28.5.dev0'` or `''`. eckit-2.0.7/python/eckit/example.py0000664000175000017500000000070415161702250017430 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. from eckit.geo import Grid grid = Grid(grid="1/1") print(grid.shape) eckit-2.0.7/python/eckit/pyproject.toml0000664000175000017500000000313515161702250020340 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. [build-system] requires = ["setuptools>=65", "Cython>=3.0", "wheel"] build-backend = "setuptools.build_meta" [project] name = "eckit" description = "Don't use this" requires-python=">=3.9" license = "Apache-2.0" license-files = ["../../LICENSE"] authors = [ { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" } ] classifiers=[ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Cython", "Programming Language :: Python :: Implementation :: CPython", "Operating System :: POSIX :: Linux", "Operating System :: MacOS", "Topic :: Scientific/Engineering", ] # dependencies = [] # NOTE don't put anything here, use setup.py instead dynamic = ["version", "dependencies"] [tool.setuptools] # NOTE perhaps add a fine-grained exclude so that we eg keep headers only # https://setuptools.pypa.io/en/latest/userguide/datafiles.html#exclude-package-data # however, beware of the interaction with data_files -- with `true` it crashes on # absolute paths etc. Fine-graining such as `tool.setuptools.package-data` may be also # needed include-package-data = false eckit-2.0.7/python/eckit/src/0000775000175000017500000000000015161702250016211 5ustar alastairalastaireckit-2.0.7/python/eckit/src/eckit/0000775000175000017500000000000015161702250017310 5ustar alastairalastaireckit-2.0.7/python/eckit/src/eckit/__init__.py0000664000175000017500000000070215161702250021420 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import findlibs findlibs.load("eckit") from eckit._eckit import * eckit-2.0.7/python/eckit/src/eckit/geo/0000775000175000017500000000000015161702250020062 5ustar alastairalastaireckit-2.0.7/python/eckit/src/eckit/geo/__init__.py0000664000175000017500000000073215161702250022175 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import findlibs findlibs.load("eckit_geo", "eckitlib") from eckit.geo._eckit_geo import * eckit-2.0.7/python/eckit/src/_eckit/0000775000175000017500000000000015161702250017447 5ustar alastairalastaireckit-2.0.7/python/eckit/src/_eckit/std.pxd0000664000175000017500000000115115161702250020754 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. from libcpp.string cimport string cdef extern from "" namespace "std" nogil: cdef cppclass ostream: pass cdef cppclass ostringstream(ostream): ostringstream() string to_string "str" () const eckit-2.0.7/python/eckit/src/_eckit/eckit.cc0000664000175000017500000000123715161702250021060 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/runtime/Main.h" void eckit_main_initialise() { if (!eckit::Main::ready()) { static const int argc = 1; static char* argv[] = {const_cast("eckit_main_initialise"), nullptr}; eckit::Main::initialise(argc, argv); } } eckit-2.0.7/python/eckit/src/_eckit/_eckit_geo.pyx0000664000175000017500000000640315161702250022304 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. cimport eckit_geo from libcpp.utility cimport pair from libcpp.vector cimport vector cimport eckit eckit.eckit_main_initialise() cdef class Area: cdef const eckit_geo.Area* _area def __cinit__(self, spec = None, **kwargs): assert bool(spec) != bool(kwargs) if kwargs or isinstance(spec, dict): from yaml import dump spec = dump(kwargs if kwargs else spec, default_flow_style=True).strip() try: assert isinstance(spec, str) self._area = eckit_geo.AreaFactory.make_from_string(spec) except RuntimeError as e: # opportunity to do something interesting raise def __eq__(self, other) -> bool: if not isinstance(other, Grid): return NotImplemented return self.spec_str == other.spec_str @property def spec_str(self) -> str: return self._area.spec_str() @property def spec(self) -> dict: from yaml import safe_load return safe_load(self.spec_str) @property def type(self) -> str: return self._area.type() cdef class Grid: cdef const eckit_geo.Grid* _grid def __cinit__(self, spec = None, **kwargs): assert bool(spec) != bool(kwargs) if kwargs or isinstance(spec, dict): from yaml import dump spec = dump(kwargs if kwargs else spec, default_flow_style=True).strip() try: assert isinstance(spec, str) self._grid = eckit_geo.GridFactory.make_from_string(spec) except RuntimeError as e: # opportunity to do something interesting raise def __eq__(self, other) -> bool: if not isinstance(other, Grid): return NotImplemented return self.spec_str == other.spec_str def to_latlons(self): cdef pair[vector[double], vector[double]] latlons = self._grid.to_latlons() return list(latlons.first), list(latlons.second) def bounding_box(self) -> tuple: cdef const eckit_geo.BoundingBox* bbox = &self._grid.boundingBox() cdef double north = bbox.north() cdef double west = bbox.west() cdef double south = bbox.south() cdef double east = bbox.east() return north, west, south, east @property def spec_str(self) -> str: return self._grid.spec_str() @property def spec(self) -> dict: from yaml import safe_load return safe_load(self.spec_str) @property def type(self) -> str: return self._grid.type() @property def uid(self) -> str: return self._grid.uid() @property def shape(self) -> tuple: cdef vector[size_t] shape_vec = self._grid.shape() return tuple(shape_vec) def size(self) -> int: return self._grid.size() def __len__(self) -> int: return self.size() def __dealloc__(self): del self._grid eckit-2.0.7/python/eckit/src/_eckit/eckit.pxd0000664000175000017500000000132015161702250021257 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. cimport std from libcpp.string cimport string cdef extern from "eckit.h": void eckit_main_initialise() cdef extern from "eckit/filesystem/PathName.h" namespace "eckit": cdef cppclass PathName: PathName(string) cdef extern from "eckit/log/JSON.h" namespace "eckit": cdef cppclass JSON: JSON(std.ostream&) eckit-2.0.7/python/eckit/src/_eckit/_eckit.pyx0000664000175000017500000000065315161702250021453 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. cimport eckit eckit.eckit_main_initialise() eckit-2.0.7/python/eckit/src/_eckit/eckit.h0000664000175000017500000000067215161702250020724 0ustar alastairalastair/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once void eckit_main_initialise(); eckit-2.0.7/python/eckit/src/_eckit/eckit_geo.pxd0000664000175000017500000000263415161702250022122 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. from libcpp.string cimport string from libcpp.utility cimport pair from libcpp.vector cimport vector cdef extern from "eckit/geo/Area.h" namespace "eckit::geo": cdef cppclass Area: string spec_str() const string type() const cdef cppclass AreaFactory: @staticmethod const Area* make_from_string(const string) except + cdef extern from "eckit/geo/area/BoundingBox.h" namespace "eckit::geo::area": cdef cppclass BoundingBox(Area): double north() const double west() const double south() const double east() const cdef extern from "eckit/geo/Grid.h" namespace "eckit::geo": cdef cppclass Grid: pair[vector[double], vector[double]] to_latlons() const string spec_str() const string type() const string uid() const vector[size_t] shape() const size_t size() const const BoundingBox& boundingBox() const cdef cppclass GridFactory: @staticmethod const Grid* make_from_string(const string) except + eckit-2.0.7/python/eckit/requirements-devel.txt0000664000175000017500000000007015161702250022000 0ustar alastairalastairCython numpy pyyaml setuptools build twine wheel pytest eckit-2.0.7/python/eckit/build_chain.sh0000775000175000017500000000363215161702250020226 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. # building & publishing of a wheel with eckit-python only, with requirements for eckitlib wheel # assumed to be executed *inside* wheelmaker docker image, or at least: # - have access to the setup_utils python package # - uv installed # - manylinux-compatible compilation stack # - env var PYVERSION, eg 3.11 # TODO unify with mir/python/mir/build_chain.sh, by moving to ci-utils/wheelmaker set -euo pipefail # prepare python rm -rf .venv uv venv --python python$PYVERSION .venv source .venv/bin/activate uv pip install --upgrade -r ./requirements-devel.txt TEST_PYPI=${TEST_PYPI:-no} if [ "$TEST_PYPI" = "yes" ] ; then EXTRA_PIP="--no-cache --index-url https://test.pypi.org/simple/" TARGET="--repository testpypi" else EXTRA_PIP="" TARGET="" fi # eckit-python prereqs # TODO get these from pyproject... if [ -n "$INSTALL_LOCALLY" ] ; then uv pip install --no-cache $INSTALL_LOCALLY/eckitlib* else uv pip install --prerelease=allow $EXTRA_PIP eckitlib fi PRF=".venv/lib/python$PYVERSION/site-packages" if [ "$(uname)" == "Darwin" ] ; then L="lib" ; else L="lib64" ; fi export ECKIT_LIB_DIR="$PRF/eckitlib/$L" export ECKIT_INCLUDE_DIRS="$PRF/eckitlib/include" # build rm -rf build dist PYTHONPATH=/buildscripts python -m build --no-isolation --wheel . # test uv pip install ./dist/* # pytest tests/ # TODO re-enable after fixed twine check dist/*whl # upload # NOTE we don't upload because of execution via ci-utils/wheelmaker/buildscripts/multirelease.sh, which uploads on its own # twine upload --verbose --skip-existing dist/*whl eckit-2.0.7/python/eckit/setup.py0000664000175000017500000000574515161702250017147 0ustar alastairalastair# (C) Copyright 2025- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import sys import warnings from os import getenv from pathlib import Path from typing import Any from Cython.Build import cythonize from setuptools import Extension from setuptools import setup source_dir = getenv("ECKIT_SOURCE_DIR", str(Path(getenv("HOME"), "git", "eckit"))) binary_dir = getenv("ECKIT_BUILD_DIR", str(Path(getenv("HOME"), "build", "eckit"))) library_dirs = getenv("ECKIT_LIB_DIR", str(Path(binary_dir, "lib"))).split(":") include_dirs_default = ":".join( str(p) for p in ( Path(source_dir, "src"), Path(binary_dir, "src"), Path(source_dir, "eckit", "src"), Path(binary_dir, "eckit", "src"), ) if p.exists() ) include_dirs = getenv("ECKIT_INCLUDE_DIRS", include_dirs_default).split(":") extra_compile_args = ["-std=c++17"] def _ext(name: str, sources: list, libraries: list) -> Extension: return Extension( name, sources, language="c++", libraries=libraries, library_dirs=library_dirs, include_dirs=include_dirs, extra_compile_args=extra_compile_args, runtime_library_dirs=library_dirs, extra_link_args=extra_compile_args, ) kwargs_set: dict[str, Any] = {} try: from setup_utils import ext_kwargs as wheel_ext_kwargs kwargs_set.update(wheel_ext_kwargs[sys.platform]) except ImportError: warnings.warn("failed to import setup_utils, won't mark the wheel as manylinux") version: str try: with open("../../VERSION", "r") as f: version = f.readlines()[0].strip() except Exception: warnings.warn("failed to read VERSION, falling back to 0.0.0") version = "0.0.0" install_requires = ["findlibs", "pyyaml"] try: import eckitlib install_requires.append(f"eckitlib=={eckitlib.__version__}") except ImportError: warnings.warn("failed to import eckitlib, not listing as a dependency") setup( name="eckit", version=version, install_requires=install_requires, ext_modules=cythonize( [ _ext( "eckit._eckit", [ "src/_eckit/_eckit.pyx", "src/_eckit/eckit.cc", ], ["eckit"], ), _ext( "eckit.geo._eckit_geo", [ "src/_eckit/_eckit_geo.pyx", "src/_eckit/eckit.cc", ], ["eckit_geo"], ), ], compiler_directives={ "language_level": 3, "c_string_type": "unicode", # accept Python str "c_string_encoding": "utf8", }, ), **kwargs_set, ) eckit-2.0.7/python/.gitignore0000664000175000017500000000664515161702250016326 0ustar alastairalastair# Cython-generated files _*.cpp _*.h # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # UV # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. #uv.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # pdm # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. # https://pdm.fming.dev/latest/usage/project/#working-with-version-control .pdm.toml .pdm-python .pdm-build/ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ # Ruff stuff: .ruff_cache/ # PyPI configuration file .pypirc # vim *swp eckit-2.0.7/python/eckit-geo-share-download.py0000775000175000017500000001156015161702250021457 0ustar alastairalastair#!/usr/bin/env python3 # Requirements: # Python >= 3.6 # PyYAML >= 6.0.1 # Requests >= 2.31.0 # # Install dependencies with pip: # pip install pyyaml requests import argparse import logging import os from pathlib import Path from urllib.parse import urlparse import requests import yaml logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") # TODO: configuration options control the cache path, available grids, but they're not considered here CONFIG_PATH = ( Path(os.path.abspath(__file__)).parents[2] / "eckit" / "share" / "eckit" / "geo" / "ORCA.yaml" ) CACHE_PATH = Path( os.getenv( "ECKIT_GEO_CACHE_PATH", Path.home() / ".local" / "share" / "eckit" / "geo" ) ) CHUNK_SIZE = 32768 def pretty_bytes(num: float, suffix="B"): """Convert bytes to a more readable format.""" for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: if abs(num) < 1024.0: return f"{num:3.1f} {unit}{suffix}" num /= 1024.0 return f"{num:.1f} Yi{suffix}" def download_file(url: str, path: Path): try: logging.info(f"Downloading '{url}'...") with requests.get(url, stream=True, timeout=60) as r: r.raise_for_status() # Verify status code path.parent.mkdir(parents=True, exist_ok=True) bytes = 0 with open(path, "wb") as f: for chunk in r.iter_content(chunk_size=CHUNK_SIZE): bytes += f.write(chunk) logging.info(f"Downloaded '{path}' ({pretty_bytes(bytes)}).") if "Content-Length" in r.headers: assert bytes == int(r.headers["Content-Length"]) return True except Exception as e: logging.error(f"Error downloading from '{url}' to '{path}': {e}") return False def is_url(string): try: result = urlparse(string) return all([result.scheme, result.netloc]) except ValueError: return False def path_from_url(url: str, cache_path: Path): path_parts = urlparse(url).path.strip("/").split("/") if path_parts and path_parts[0] == "repository": path_parts.remove("repository") return cache_path.joinpath(*path_parts) def execute(args): with open(args.config, "r") as file: config = yaml.safe_load(file) # Build lookup dict from grid_names # Each entry is a flat dict where the grid name key has None value (due to YAML anchor) # and the rest of the keys are grid properties grids_by_name = {} for entry in config.get("grid_names", []): if isinstance(entry, dict): # Find the grid name (key with None value, which is the anchor) grid_name = None for key, value in entry.items(): if value is None: grid_name = key break if grid_name: # The grid info is all the other keys in the entry grid_info = {k: v for k, v in entry.items() if k != grid_name} grids_by_name[grid_name] = grid_info # Handle --list-grids if args.list_grids: print("Known grid names:") for name in sorted(grids_by_name.keys()): grid = grids_by_name[name] uid = grid.get("orca_uid", "") print(f" {name}" + (f" (uid: {uid})" if uid else "")) return # Determine which grids to process if args.grid == ["all"]: grids_to_process = list(grids_by_name.values()) else: grids_to_process = [] for key in args.grid: grid = grids_by_name.get(key) if not grid: logging.error(f"Grid '{key}' is not a known grid name") continue grids_to_process.append(grid) urls = set() for grid in grids_to_process: url = grid.get("url") if url and is_url(url): urls.add(url) for url in sorted(urls): path = path_from_url(url, args.cache_path) if path.exists() and not args.overwrite: logging.info(f"File already exists: '{path}'") continue if download_file(url, path): assert path.exists() if __name__ == "__main__": p = argparse.ArgumentParser(description="Create binary grid data files") p.add_argument( "--list-grids", action="store_true", help="List all known grid names and UIDs, then exit." ) p.add_argument( "--config", default=CONFIG_PATH, help=f"Path to the YAML configuration file (default: {CONFIG_PATH})", ) p.add_argument( "--cache-path", default=CACHE_PATH, help=f"Path to the cache directory (default: {CACHE_PATH})", ) p.add_argument( "--grid", default=["all"], nargs="+", help="Grid to cache (default: all)" ) p.add_argument("--overwrite", action="store_true", help="Overwrite existing files") execute(p.parse_args()) eckit-2.0.7/README.md0000664000175000017500000000571215161702250014266 0ustar alastairalastair# EcKit
[![Latest Tag](https://img.shields.io/github/v/tag/ecmwf/eckit?label=latest&style=flat-square&logo=github)](https://github.com/ecmwf/eckit/tags) [![CI](https://img.shields.io/github/actions/workflow/status/ecmwf/eckit/ci.yml?branch=master&label=ci%20(master)&style=flat-square&logo=github)](https://github.com/ecmwf/eckit/actions/workflows/ci.yml) [![CI](https://img.shields.io/github/actions/workflow/status/ecmwf/eckit/ci.yml?branch=develop&label=ci%20(develop)&style=flat-square&logo=github)](https://github.com/ecmwf/eckit/actions/workflows/ci.yml) [![codecov](https://codecov.io/gh/ecmwf/eckit/branch/develop/graph/badge.svg)](https://codecov.io/gh/ecmwf/eckit) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/ecmwf/eckit/blob/develop/LICENSE) [![Project Maturity](https://github.com/ecmwf/codex/raw/refs/heads/main/Project%20Maturity/graduated_badge.svg)](https://github.com/ecmwf/codex/raw/refs/heads/main/Project%20Maturity#graduated)
> [!IMPORTANT] > This software is **Graduated** and subject to ECMWF's guidelines on > [Software Maturity](https://github.com/ecmwf/codex/raw/refs/heads/main/Project%20Maturity). ## Overview EcKit is a cross-platform C++ toolkit that supports development of tools and applications at ECMWF. It is based on code developed over the last 20 years within the MARS software and was re-factored out to be reused by other applications. It provides an abstraction layer on top of the operating system, so it is easier to port code to new architectures. It is developed taking into account the robustness requirements of running production systems at ECMWF. The main focus is UNIX/POSIX systems, and it has been thoroughly tested on Linux and Mac OSX. Historically, the code base pre-dates and in some way can be seen as a leaner substitute for some 'Boost' libraries. ## Features It features facilities to easily handle, in a cross-platform way: * multi-threading * json and yaml parsing * object serialization and persistence * object serialization supporting compression, to and from file and network * configuration of user options and resources * file-system abstraction * regular expressions * socket networking * http protocol * type-to-type conversions * asynchronous IO * asynchronous processing * exception handling and stack dumping * MPI object-oriented wrapper * linear algebra abstraction with multiple backends (BLAS, MKL, Eigen3) * advanced container classes * space partition trees * file-mapped arrays ## Requirements Make sure ecbuild is installed and the ecbuild executable script is found (`which ecbuild`). Now proceed with installation as follows: ```bash # Environment --- Edit as needed srcdir=$(pwd) builddir=build installdir=$HOME/.local # 1. Create the build directory: mkdir $builddir cd $builddir # 2. Run CMake ecbuild --prefix=$installdir -- $srcdir # 3. Compile / Install make -j10 make install # 4. Check installation $installdir/bin/eckit-version ``` eckit-2.0.7/ChangeLog0000664000175000017500000000000015161702250014542 0ustar alastairalastaireckit-2.0.7/format-sources.sh0000775000175000017500000000742515161702250016322 0ustar alastairalastair#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REQUIRED_VERSION=19 usage() { cat <<'EOF' Usage: format-sources.sh [OPTIONS] [FILE ...] Apply clang-format to eckit source files. The formatter binary is resolved in this order: 1. $CLANG_FORMAT environment variable (must be an executable path) 2. clang-format-19 on $PATH 3. clang-format on $PATH Unless --any-version is given, the binary must report major version 19. When no files are given, recursively formats all *.h and *.cc files relative to the script's location (the repository root), excluding any contrib/ directories. Formatting runs in parallel using all available cores. The script can be invoked from any working directory. Options: -h, --help Show this help message and exit --any-version Skip the version-19 check (prints a warning) Environment: CLANG_FORMAT Explicit path to the clang-format binary to use Examples: ./format-sources.sh # format all ./format-sources.sh src/eckit/foo.cc # one file CLANG_FORMAT=/usr/bin/clang-format-19 ./format-sources.sh # explicit binary ./format-sources.sh --any-version src/eckit/bar.h # any version EOF } # Check that the resolved binary is clang-format $REQUIRED_VERSION. # Usage: check_version check_version() { local binary="$1" local version_output major version_output=$("$binary" --version 2>&1) || { echo "Error: failed to run '$binary --version'" >&2 exit 1 } # Typical output: "clang-format version 19.1.7 (...)" major=$(echo "$version_output" | sed -n 's/.*version \([0-9]\+\).*/\1/p') if [[ -z "$major" ]]; then echo "Error: unable to parse version from '$binary --version' output:" >&2 echo " $version_output" >&2 exit 1 fi if [[ "$major" -ne "$REQUIRED_VERSION" ]]; then echo "Error: clang-format version $major found, version $REQUIRED_VERSION required" >&2 echo " binary: $binary" >&2 echo " Set CLANG_FORMAT to the correct binary or use --any-version to skip this check." >&2 exit 1 fi } any_version=false while [[ $# -gt 0 ]]; do case "$1" in -h|--help) usage exit 0 ;; --any-version) any_version=true shift ;; --) shift break ;; -*) echo "Error: unknown option '$1'" >&2 usage >&2 exit 1 ;; *) break ;; esac done # Resolve clang-format binary if [[ -n "${CLANG_FORMAT:-}" ]]; then if [[ ! -x "$CLANG_FORMAT" ]]; then echo "Error: CLANG_FORMAT is set to '$CLANG_FORMAT' but it is not executable" >&2 exit 1 fi elif command -v clang-format-19 &>/dev/null; then CLANG_FORMAT="$(command -v clang-format-19)" elif command -v clang-format &>/dev/null; then CLANG_FORMAT="$(command -v clang-format)" else echo "Error: no clang-format found" >&2 echo " Install clang-format-19, add it to \$PATH, or set the CLANG_FORMAT variable." >&2 exit 1 fi if [[ "$any_version" == true ]]; then echo "Warning: --any-version used; skipping version check for $CLANG_FORMAT" >&2 else check_version "$CLANG_FORMAT" fi # Collect files: positional args, or default to all *.h / *.cc in the repo if [[ $# -gt 0 ]]; then files=("$@") else mapfile -t files < <(find "$SCRIPT_DIR" \( -name '*.h' -o -name '*.cc' \) \ -not -path '*/contrib/*' -type f) if [[ ${#files[@]} -eq 0 ]]; then echo "No source files found" >&2 exit 1 fi fi printf '%s\0' "${files[@]}" \ | xargs -0 -P0 -n1 "$CLANG_FORMAT" -i -style=file -fallback-style=none eckit-2.0.7/.clang-format0000664000175000017500000000544015161702250015360 0ustar alastairalastairLanguage: Cpp AccessModifierOffset: -4 AlignAfterOpenBracket: Align AlignConsecutiveAssignments: true AlignConsecutiveDeclarations: false AlignEscapedNewlines: Left AlignOperands: true AlignTrailingComments: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: true AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: Inline AllowShortIfStatementsOnASingleLine: false AllowShortLoopsOnASingleLine: false AllowShortEnumsOnASingleLine: false AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: true BinPackArguments: true BinPackParameters: true BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false AfterFunction: false AfterNamespace: false AfterObjCDeclaration: false AfterStruct: false AfterUnion: false BeforeCatch: true BeforeElse: true IndentBraces: false SplitEmptyFunction: true SplitEmptyRecord: true SplitEmptyNamespace: true BreakAfterReturnType: Automatic BreakBeforeBinaryOperators: None BreakBeforeBraces: Custom BreakBeforeInheritanceComma: false BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false BreakConstructorInitializers: AfterColon BreakAfterJavaFieldAnnotations: false BreakStringLiterals: true ColumnLimit: 120 CommentPragmas: "^ IWYU pragma:" CompactNamespaces: false ConstructorInitializerAllOnOneLineOrOnePerLine: true ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false DisableFormat: false ExperimentalAutoDetectBinPacking: false EmptyLineBeforeAccessModifier: Always EmptyLineAfterAccessModifier: Always FixNamespaceComments: true ForEachMacros: - foreach - Q_FOREACH - BOOST_FOREACH IncludeCategories: - Regex: '^<.*\.h>' Priority: 1 - Regex: "^<.*" Priority: 2 - Regex: ".*" Priority: 3 IncludeIsMainRegex: "([-_](test|unittest))?$" IndentCaseLabels: true IndentWidth: 4 IndentWrappedFunctionNames: false KeepEmptyLinesAtTheStartOfBlocks: true MacroBlockBegin: "" MacroBlockEnd: "" MaxEmptyLinesToKeep: 2 NamespaceIndentation: None PenaltyBreakAssignment: 2 PenaltyBreakBeforeFirstCallParameter: 1 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left ReflowComments: true SortIncludes: true SortUsingDeclarations: true SpaceAfterCStyleCast: false SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 2 SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false Standard: Auto TabWidth: 4 UseTab: Never eckit-2.0.7/.travis.yml0000664000175000017500000001434215161702250015117 0ustar alastairalastairlanguage: cpp cache: directories: - ${HOME}/deps/cmake # - ${HOME}/deps/openmpi # - ${HOME}/deps/mpich jobs: include: - os: linux arch: amd64 compiler: clang env: - CACHE_NAME=linux-clang38 - CXX_COMPILER='clang++-3.8' C_COMPILER='clang-3.8' - ECKIT_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DENABLE_EXPERIMENTAL=OFF -DENABLE_MPI=OFF" addons: apt: sources: ['llvm-toolchain-precise', 'ubuntu-toolchain-r-test'] packages: ['clang-3.8'] - os: linux arch: ppc64le compiler: clang env: - CACHE_NAME=linux-clang38 - CXX_COMPILER='clang++-3.8' C_COMPILER='clang-3.8' - ECKIT_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DENABLE_EXPERIMENTAL=OFF -DENABLE_MPI=OFF" - ECKIT_CTEST_OPTIONS="-E eckit_test_memory_mmap" addons: apt: sources: ['llvm-toolchain-precise', 'ubuntu-toolchain-r-test'] packages: ['clang-3.8'] - os: linux arch: amd64 compiler: gcc env: - CACHE_NAME=linux-gcc5 - CXX_COMPILER='g++-5' C_COMPILER='gcc-5' - ECKIT_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DENABLE_EXPERIMENTAL=OFF -DENABLE_MPI=OFF" addons: apt: sources: ['ubuntu-toolchain-r-test'] packages: ['g++-5', 'gcc-5'] - os: linux arch: ppc64le compiler: gcc env: - CACHE_NAME=linux-gcc5 - CXX_COMPILER='g++-5' C_COMPILER='gcc-5' - ECKIT_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DENABLE_EXPERIMENTAL=OFF -DENABLE_MPI=OFF" - ECKIT_CTEST_OPTIONS="-E eckit_test_memory_mmap" addons: apt: sources: ['ubuntu-toolchain-r-test'] packages: ['g++-5', 'gcc-5'] - os: linux arch: amd64 compiler: gcc env: - CACHE_NAME=linux-gcc7 - CXX_COMPILER='g++-7' C_COMPILER='gcc-7' - ECKIT_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DENABLE_EXPERIMENTAL=OFF -DENABLE_GPROF=ON -DENABLE_MPI=OFF" addons: apt: sources: ['ubuntu-toolchain-r-test'] packages: ['g++-7', 'gcc-7'] - os: linux arch: ppc64le compiler: gcc env: - CACHE_NAME=linux-gcc7 - CXX_COMPILER='g++-7' C_COMPILER='gcc-7' - ECKIT_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DENABLE_EXPERIMENTAL=OFF -DENABLE_MPI=OFF" - ECKIT_CTEST_OPTIONS="-E eckit_test_memory_mmap" addons: apt: sources: ['ubuntu-toolchain-r-test'] packages: ['g++-7', 'gcc-7'] - os: osx env: - CACHE_NAME=osx-clang - CXX_COMPILER='clang++' C_COMPILER='clang' - ECKIT_CMAKE_OPTIONS="-DCMAKE_BUILD_TYPE=DEBUG -DENABLE_EXPERIMENTAL=OFF -DENABLE_MPI=OFF" osx_image: xcode12 before_install: # Fixes to pre-installed packages # - | # ### Fix pre-installed packages # if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then # brew update # brew list oclint || brew cask uninstall oclint # Prevent conflict with gcc # fi # Set compilers - | ### Set compilers export CC=${C_COMPILER} export CXX=${CXX_COMPILER} install: # All dependencies are installed in ${TRAVIS_BUILD_DIR}/deps/ - DEPS_DIR=${HOME}/deps - mkdir -p ${DEPS_DIR} && cd ${DEPS_DIR} # Install compilers if needed -- only for fortran # - | # ### Install gcc (homebrew) # if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then # brew upgrade gcc || brew install gcc # fi # # Install MPI # - | # ### Install MPI # install-mpi.sh ${MPI} # source ${DEPS_DIR}/${MPI}/env.sh # echo "${MPI_HOME}" # echo "${PATH}" # Install CMake - | ### Install CMake #sudo apt-get install libuv1-dbg libuv1-dev libuv1 rhash libstdc++6; if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then CMAKE_VERSION=3.18.4 if [[ -z "$(ls -A ${DEPS_DIR}/cmake)" ]]; then if [[ "${TRAVIS_ARCH}" == "amd64" ]]; then CMAKE_URL="https://cmake.org/files/v${CMAKE_VERSION%.*}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" mkdir -p ${DEPS_DIR}/cmake && travis_retry wget --no-check-certificate --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR}/cmake fi if [[ "${TRAVIS_ARCH}" = "ppc64le" ]]; then # Build from source CMAKE_URL="https://cmake.org/files/v${CMAKE_VERSION%.*}/cmake-${CMAKE_VERSION}.tar.gz" mkdir -p ${DEPS_DIR}/cmake-source && travis_retry wget --quiet -O - ${CMAKE_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR}/cmake-source ( cd ${DEPS_DIR}/cmake-source mkdir -p build && cd build ../bootstrap --prefix=${DEPS_DIR}/cmake && make -j4 install ) fi else echo "CMake already found in cache" fi export PATH=${DEPS_DIR}/cmake/bin:${PATH} else brew upgrade cmake || brew install cmake brew install openssl export OPENSSL_ROOT_DIR=/usr/local/opt/openssl fi cmake --version # Install ecbuild - | ### Install ecbuild git clone --depth 1 -b develop https://github.com/ecmwf/ecbuild ${DEPS_DIR}/ecbuild export ecbuild_ROOT=${DEPS_DIR}/ecbuild ${ecbuild_ROOT}/bin/ecbuild --version script: # Environment variables - echo ${CXX} - echo ${CC} # - echo ${FC} # - echo ${MPI_HOME} - echo ${PATH} - | ECKIT_SOURCE_DIR=${TRAVIS_BUILD_DIR} ECKIT_BUILD_DIR=${TRAVIS_BUILD_DIR}/builds/eckit # build eckit - mkdir -p ${ECKIT_BUILD_DIR} && cd ${ECKIT_BUILD_DIR} - cmake ${ECKIT_CMAKE_OPTIONS} ${ECKIT_SOURCE_DIR} - make -j4 - bin/eckit-version # test eckit - ctest ${ECKIT_CTEST_OPTIONS} after_success: - | if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then # Creating report cd ${ECKIT_BUILD_DIR} lcov --directory . --capture --output-file coverage.info # capture coverage info lcov --remove coverage.info '/usr/*' --output-file coverage.info # filter out system lcov --list coverage.info #debug info # Uploading report to CodeCov bash <(curl -s https://codecov.io/bash) -t 33404098-c4e6-4701-a89c-36f33ceb3795 || echo "Codecov did not collect coverage reports" fi after_failure: - cd ${ECKIT_BUILD_DIR} - ctest -VV --rerun-failed ${ECKIT_CTEST_OPTIONS} - cat ecbuild.log eckit-2.0.7/docs/0000775000175000017500000000000015161702250013732 5ustar alastairalastaireckit-2.0.7/docs/requirements.txt0000664000175000017500000000012715161702250017216 0ustar alastairalastairsphinx pydata_sphinx_theme sphinx-copybutton sphinx-tabs sphinxcontrib-mermaid breathe eckit-2.0.7/docs/pyproject.toml0000664000175000017500000000003715161702250016646 0ustar alastairalastair[tool.black] line-length = 120 eckit-2.0.7/docs/conf.py0000664000175000017500000000571715161702250015243 0ustar alastairalastair# Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html # -- Imports ----------------------------------------------------------------- import sys import os import datetime import re import subprocess # -- Path manipulation-------------------------------------------------------- sys.path.append(os.path.abspath("../tests")) # -- Run Doxygen ------------------------------------------------------------- # Generate Doxygen documentation in the XML format. subprocess.check_call("doxygen Doxyfile", shell=True) # -- Project information ----------------------------------------------------- project = "EcKit" author = "ECMWF" year = datetime.datetime.now().year years = "1996-%s" % (year,) copyright = "%s, %s" % (years, author) def parse_version(ver_str): return re.sub("^((([0-9]+)\\.)+([0-9]+)).*", "\\1", ver_str) here = os.path.abspath(os.path.dirname(__file__)) # Get the current version directly from the source. with open(os.path.join(here, "..", "VERSION"), "r") as f: release = f.readline().strip() # full version string version = parse_version(release) # feature version # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ "pydata_sphinx_theme", "breathe", "sphinx.ext.autosectionlabel", "sphinxcontrib.mermaid", "sphinx_copybutton", ] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] source_suffix = ".rst" master_doc = "index" pygments_style = "sphinx" highlight_language = "c++" # -- Options for HTML output ------------------------------------------------- html_theme = "pydata_sphinx_theme" # Do not show source link on each page html_show_sourcelink = False html_sidebars = { "**": [], } html_theme_options = { "navbar_align": "left", "navbar_start": ["navbar-logo"], "navbar_center": ["navbar-nav"], "navbar_end": ["navbar-icon-links", "theme-switcher", "version-switcher"], "navbar_persistent": ["search-button"], "primary_sidebar_end": [], # On local builds no version.json is present "check_switcher": False, "logo": {"text": "EcKit"}, } html_context = { # Enable auto detection of light/dark mode "default_mode": "auto" } # -- Breathe configuration --------------------------------------------------- breathe_projects = { "ECKIT": os.path.abspath("_build/xml") # Path to generated XML } breathe_default_project = "ECKIT" eckit-2.0.7/docs/index.rst0000664000175000017500000000027215161702250015574 0ustar alastairalastairWelcome to EcKit’s Documentation! ================================= .. index:: Contents .. toctree:: :maxdepth: 1 :caption: Contents content/concepts content/reference eckit-2.0.7/docs/Makefile0000664000175000017500000000132215161702250015370 0ustar alastairalastair#!/usr/bin/env make -f # Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= -W --keep-going # turn warnings into errors, run to completion SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) eckit-2.0.7/docs/Doxyfile0000664000175000017500000040017315161702250015445 0ustar alastairalastair# Doxyfile 1.13.2 # This file describes the settings to be used by the documentation system # Doxygen (www.doxygen.org) for a project. # # All text after a double hash (##) is considered a comment and is placed in # front of the TAG it is preceding. # # All text after a single hash (#) is considered a comment and will be ignored. # The format is: # TAG = value [value, ...] # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). # # Note: # # Use Doxygen to compare the used configuration file with the template # configuration file: # doxygen -x [configFile] # Use Doxygen to compare the used configuration file with the template # configuration file without replacing the environment variables or CMake type # replacement variables: # doxygen -x_noenv [configFile] #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # This tag specifies the encoding used for all characters in the configuration # file that follow. The default is UTF-8 which is also the encoding used for all # text before the first occurrence of this tag. Doxygen uses libiconv (or the # iconv built into libc) for the transcoding. See # https://www.gnu.org/software/libiconv/ for the list of possible encodings. # The default value is: UTF-8. DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded by # double-quotes, unless you are using Doxywizard) that should identify the # project for which the documentation is generated. This name is used in the # title of most generated pages and in a few other places. # The default value is: My Project. PROJECT_NAME = "ECKIT" # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version # control system is used. PROJECT_NUMBER = # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewers a # quick idea about the purpose of the project. Keep the description short. PROJECT_BRIEF = # With the PROJECT_LOGO tag one can specify a logo or an icon that is included # in the documentation. The maximum height of the logo should not exceed 55 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy # the logo to the output directory. PROJECT_LOGO = # With the PROJECT_ICON tag one can specify an icon that is included in the tabs # when the HTML document is shown. Doxygen will copy the logo to the output # directory. PROJECT_ICON = # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path # into which the generated documentation will be written. If a relative path is # entered, it will be relative to the location where Doxygen was started. If # left blank the current directory will be used. OUTPUT_DIRECTORY = _build # If the CREATE_SUBDIRS tag is set to YES then Doxygen will create up to 4096 # sub-directories (in 2 levels) under the output directory of each output format # and will distribute the generated files over these directories. Enabling this # option can be useful when feeding Doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise cause # performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to # control the number of sub-directories. # The default value is: NO. CREATE_SUBDIRS = NO # Controls the number of sub-directories that will be created when # CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every # level increment doubles the number of directories, resulting in 4096 # directories at level 8 which is the default and also the maximum value. The # sub-directories are organized in 2 levels, the first level always has a fixed # number of 16 directories. # Minimum value: 0, maximum value: 8, default value: 8. # This tag requires that the tag CREATE_SUBDIRS is set to YES. CREATE_SUBDIRS_LEVEL = 8 # If the ALLOW_UNICODE_NAMES tag is set to YES, Doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode # U+3044. # The default value is: NO. ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by Doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, # Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English # (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, # Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with # English messages), Korean, Korean-en (Korean with English messages), Latvian, # Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, # Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, # Swedish, Turkish, Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English # If the BRIEF_MEMBER_DESC tag is set to YES, Doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. # The default value is: YES. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES, Doxygen will prepend the brief # description of a member or function before the detailed description # # Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. # The default value is: YES. REPEAT_BRIEF = YES # This tag implements a quasi-intelligent brief description abbreviator that is # used to form the text in various listings. Each string in this list, if found # as the leading text of the brief description, will be stripped from the text # and the result, after processing the whole list, is used as the annotated # text. Otherwise, the brief description is used as-is. If left blank, the # following values are used ($name is automatically replaced with the name of # the entity):The $name class, The $name widget, The $name file, is, provides, # specifies, contains, represents, a, an and the. ABBREVIATE_BRIEF = "The $name class" \ "The $name widget" \ "The $name file" \ is \ provides \ specifies \ contains \ represents \ a \ an \ the # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # Doxygen will generate a detailed section even if there is only a brief # description. # The default value is: NO. ALWAYS_DETAILED_SEC = NO # If the INLINE_INHERITED_MEMB tag is set to YES, Doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. # The default value is: NO. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES, Doxygen will prepend the full path # before files name in the file list and in the header files. If set to NO the # shortest path that makes the file name unique will be used # The default value is: YES. FULL_PATH_NAMES = YES # The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. # Stripping is only done if one of the specified strings matches the left-hand # part of the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which Doxygen is run is used as the path to # strip. # # Note that you can specify absolute paths here, but also relative paths, which # will be relative from the directory where Doxygen is started. # This tag requires that the tag FULL_PATH_NAMES is set to YES. STRIP_FROM_PATH = # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the # path mentioned in the documentation of a class, which tells the reader which # header file to include in order to use a class. If left blank only the name of # the header file containing the class definition is used. Otherwise one should # specify the list of include paths that are normally passed to the compiler # using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, Doxygen will generate much shorter (but # less readable) file names. This can be useful if your file system doesn't # support long names like on DOS, Mac, or CD-ROM. # The default value is: NO. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen will interpret the # first line (until the first dot, question mark or exclamation mark) of a # Javadoc-style comment as the brief description. If set to NO, the Javadoc- # style will behave just like regular Qt-style comments (thus requiring an # explicit @brief command for a brief description.) # The default value is: NO. JAVADOC_AUTOBRIEF = NO # If the JAVADOC_BANNER tag is set to YES then Doxygen will interpret a line # such as # /*************** # as being the beginning of a Javadoc-style comment "banner". If set to NO, the # Javadoc-style will behave just like regular comments and it will not be # interpreted by Doxygen. # The default value is: NO. JAVADOC_BANNER = NO # If the QT_AUTOBRIEF tag is set to YES then Doxygen will interpret the first # line (until the first dot, question mark or exclamation mark) of a Qt-style # comment as the brief description. If set to NO, the Qt-style will behave just # like regular Qt-style comments (thus requiring an explicit \brief command for # a brief description.) # The default value is: NO. QT_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen treat a # multi-line C++ special comment block (i.e. a block of //! or /// comments) as # a brief description. This used to be the default behavior. The new default is # to treat a multi-line C++ comment block as a detailed description. Set this # tag to YES if you prefer the old behavior instead. # # Note that setting this tag to YES also means that rational rose comments are # not recognized any more. # The default value is: NO. MULTILINE_CPP_IS_BRIEF = NO # By default Python docstrings are displayed as preformatted text and Doxygen's # special commands cannot be used. By setting PYTHON_DOCSTRING to NO the # Doxygen's special commands can be used and the contents of the docstring # documentation blocks is shown as Doxygen documentation. # The default value is: YES. PYTHON_DOCSTRING = YES # If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the # documentation from any documented member that it re-implements. # The default value is: YES. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES then Doxygen will produce a new # page for each member. If set to NO, the documentation of a member will be part # of the file/class/namespace that contains it. # The default value is: NO. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen # uses this value to replace tabs by spaces in code fragments. # Minimum value: 1, maximum value: 16, default value: 4. TAB_SIZE = 4 # This tag can be used to specify a number of aliases that act as commands in # the documentation. An alias has the form: # name=value # For example adding # "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading # "Side Effects:". Note that you cannot put \n's in the value part of an alias # to insert newlines (in the resulting output). You can put ^^ in the value part # of an alias to insert a newline as if a physical newline was in the original # file. When you need a literal { or } or , in the value part of an alias you # have to escape them by means of a backslash (\), this can lead to conflicts # with the commands \{ and \} for these it is advised to use the version @{ and # @} or use a double escape (\\{ and \\}) ALIASES = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For # instance, some of the names that are used will be different. The list of all # members will be omitted, etc. # The default value is: NO. OPTIMIZE_OUTPUT_FOR_C = NO # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or # Python sources only. Doxygen will then generate output that is more tailored # for that language. For instance, namespaces will be presented as packages, # qualified scopes will look different, etc. # The default value is: NO. OPTIMIZE_OUTPUT_JAVA = NO # Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran # sources. Doxygen will then generate output that is tailored for Fortran. # The default value is: NO. OPTIMIZE_FOR_FORTRAN = NO # Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL # sources. Doxygen will then generate output that is tailored for VHDL. # The default value is: NO. OPTIMIZE_OUTPUT_VHDL = NO # Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice # sources only. Doxygen will then generate output that is more tailored for that # language. For instance, namespaces will be presented as modules, types will be # separated into more groups, etc. # The default value is: NO. OPTIMIZE_OUTPUT_SLICE = NO # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by Doxygen: IDL, Java, JavaScript, # Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, # VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files). For instance to make Doxygen treat .inc files # as Fortran files (default is PHP), and .f files as C (default is Fortran), # use: inc=Fortran f=C. # # Note: For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise # the files are not read by Doxygen. When specifying no_extension you should add # * to the FILE_PATTERNS. # # Note see also the list of default file extension mappings. EXTENSION_MAPPING = # If the MARKDOWN_SUPPORT tag is enabled then Doxygen pre-processes all comments # according to the Markdown format, which allows for more readable # documentation. See https://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by Doxygen, so you can # mix Doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. # The default value is: YES. MARKDOWN_SUPPORT = YES # When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up # to that level are automatically included in the table of contents, even if # they do not have an id attribute. # Note: This feature currently applies only to Markdown headings. # Minimum value: 0, maximum value: 99, default value: 6. # This tag requires that the tag MARKDOWN_SUPPORT is set to YES. TOC_INCLUDE_HEADINGS = 6 # The MARKDOWN_ID_STYLE tag can be used to specify the algorithm used to # generate identifiers for the Markdown headings. Note: Every identifier is # unique. # Possible values are: DOXYGEN use a fixed 'autotoc_md' string followed by a # sequence number starting at 0 and GITHUB use the lower case version of title # with any whitespace replaced by '-' and punctuation characters removed. # The default value is: DOXYGEN. # This tag requires that the tag MARKDOWN_SUPPORT is set to YES. MARKDOWN_ID_STYLE = DOXYGEN # When enabled Doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by putting a % sign in front of the word or # globally by setting AUTOLINK_SUPPORT to NO. Words listed in the # AUTOLINK_IGNORE_WORDS tag are excluded from automatic linking. # The default value is: YES. AUTOLINK_SUPPORT = YES # This tag specifies a list of words that, when matching the start of a word in # the documentation, will suppress auto links generation, if it is enabled via # AUTOLINK_SUPPORT. This list does not affect affect links explicitly created # using \# or the \link or commands. # This tag requires that the tag AUTOLINK_SUPPORT is set to YES. AUTOLINK_IGNORE_WORDS = # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want # to include (a tag file for) the STL sources as input, then you should set this # tag to YES in order to let Doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); # versus func(std::string) {}). This also makes the inheritance and # collaboration diagrams that involve STL classes more complete and accurate. # The default value is: NO. BUILTIN_STL_SUPPORT = NO # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. # The default value is: NO. CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: # https://www.riverbankcomputing.com/software) sources only. Doxygen will parse # them like normal C++ but will assume all classes use public instead of private # inheritance when no explicit protection keyword is present. # The default value is: NO. SIP_SUPPORT = NO # For Microsoft's IDL there are propget and propput attributes to indicate # getter and setter methods for a property. Setting this option to YES will make # Doxygen to replace the get and set methods by a property in the documentation. # This will only work if the methods are indeed getting or setting a simple # type. If this is not the case, or you want to show the methods anyway, you # should set this option to NO. # The default value is: YES. IDL_PROPERTY_SUPPORT = YES # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES then Doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. # The default value is: NO. DISTRIBUTE_GROUP_DOC = NO # If one adds a struct or class to a group and this option is enabled, then also # any nested class or struct is added to the same group. By default this option # is disabled and one has to add nested compounds explicitly via \ingroup. # The default value is: NO. GROUP_NESTED_COMPOUNDS = NO # Set the SUBGROUPING tag to YES to allow class member groups of the same type # (for instance a group of public functions) to be put as a subgroup of that # type (e.g. under the Public Functions section). Set it to NO to prevent # subgrouping. Alternatively, this can be done per class using the # \nosubgrouping command. # The default value is: YES. SUBGROUPING = YES # When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions # are shown inside the group in which they are included (e.g. using \ingroup) # instead of on a separate page (for HTML and Man pages) or section (for LaTeX # and RTF). # # Note that this feature does not work in combination with # SEPARATE_MEMBER_PAGES. # The default value is: NO. INLINE_GROUPED_CLASSES = NO # When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions # with only public data fields or simple typedef fields will be shown inline in # the documentation of the scope in which they are defined (i.e. file, # namespace, or group documentation), provided this scope is documented. If set # to NO, structs, classes, and unions are shown on a separate page (for HTML and # Man pages) or section (for LaTeX and RTF). # The default value is: NO. INLINE_SIMPLE_STRUCTS = NO # When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or # enum is documented as struct, union, or enum with the name of the typedef. So # typedef struct TypeS {} TypeT, will appear in the documentation as a struct # with name TypeT. When disabled the typedef will appear as a member of a file, # namespace, or class. And the struct will be named TypeS. This can typically be # useful for C code in case the coding convention dictates that all compound # types are typedef'ed and only the typedef is referenced, never the tag name. # The default value is: NO. TYPEDEF_HIDES_STRUCT = NO # The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This # cache is used to resolve symbols given their name and scope. Since this can be # an expensive process and often the same symbol appears multiple times in the # code, Doxygen keeps a cache of pre-resolved symbols. If the cache is too small # Doxygen will become slower. If the cache is too large, memory is wasted. The # cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range # is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 # symbols. At the end of a run Doxygen will report the cache usage and suggest # the optimal cache size from a speed point of view. # Minimum value: 0, maximum value: 9, default value: 0. LOOKUP_CACHE_SIZE = 0 # The NUM_PROC_THREADS specifies the number of threads Doxygen is allowed to use # during processing. When set to 0 Doxygen will based this on the number of # cores available in the system. You can set it explicitly to a value larger # than 0 to get more control over the balance between CPU load and processing # speed. At this moment only the input processing can be done using multiple # threads. Since this is still an experimental feature the default is set to 1, # which effectively disables parallel processing. Please report any issues you # encounter. Generating dot graphs in parallel is controlled by the # DOT_NUM_THREADS setting. # Minimum value: 0, maximum value: 32, default value: 1. NUM_PROC_THREADS = 0 # If the TIMESTAMP tag is set different from NO then each generated page will # contain the date or date and time when the page was generated. Setting this to # NO can help when comparing the output of multiple runs. # Possible values are: YES, NO, DATETIME and DATE. # The default value is: NO. TIMESTAMP = NO #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES, Doxygen will assume all entities in # documentation are documented, even if no documentation was available. Private # class members and static file members will be hidden unless the # EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. # Note: This will also disable the warnings about undocumented members that are # normally produced when WARNINGS is set to YES. # The default value is: NO. EXTRACT_ALL = YES # If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will # be included in the documentation. # The default value is: NO. EXTRACT_PRIVATE = NO # If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual # methods of a class will be included in the documentation. # The default value is: NO. EXTRACT_PRIV_VIRTUAL = NO # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal # scope will be included in the documentation. # The default value is: NO. EXTRACT_PACKAGE = NO # If the EXTRACT_STATIC tag is set to YES, all static members of a file will be # included in the documentation. # The default value is: NO. EXTRACT_STATIC = NO # If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined # locally in source files will be included in the documentation. If set to NO, # only classes defined in header files are included. Does not have any effect # for Java sources. # The default value is: YES. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. If set to YES, local methods, # which are defined in the implementation section but not in the interface are # included in the documentation. If set to NO, only methods in the interface are # included. # The default value is: NO. EXTRACT_LOCAL_METHODS = NO # If this flag is set to YES, the members of anonymous namespaces will be # extracted and appear in the documentation as a namespace called # 'anonymous_namespace{file}', where file will be replaced with the base name of # the file that contains the anonymous namespace. By default anonymous namespace # are hidden. # The default value is: NO. EXTRACT_ANON_NSPACES = NO # If this flag is set to YES, the name of an unnamed parameter in a declaration # will be determined by the corresponding definition. By default unnamed # parameters remain unnamed in the output. # The default value is: YES. RESOLVE_UNNAMED_PARAMS = YES # If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation # section is generated. This option has no effect if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO, these classes will be included in the various overviews. This option # will also hide undocumented C++ concepts if enabled. This option has no effect # if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO # If the HIDE_UNDOC_NAMESPACES tag is set to YES, Doxygen will hide all # undocumented namespaces that are normally visible in the namespace hierarchy. # If set to NO, these namespaces will be included in the various overviews. This # option has no effect if EXTRACT_ALL is enabled. # The default value is: YES. HIDE_UNDOC_NAMESPACES = YES # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all friend # declarations. If set to NO, these declarations will be included in the # documentation. # The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any # documentation blocks found inside the body of a function. If set to NO, these # blocks will be appended to the function's detailed documentation block. # The default value is: NO. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation that is typed after a # \internal command is included. If the tag is set to NO then the documentation # will be excluded. Set it to YES to include the internal documentation. # The default value is: NO. INTERNAL_DOCS = NO # With the correct setting of option CASE_SENSE_NAMES Doxygen will better be # able to match the capabilities of the underlying filesystem. In case the # filesystem is case sensitive (i.e. it supports files in the same directory # whose names only differ in casing), the option must be set to YES to properly # deal with such files in case they appear in the input. For filesystems that # are not case sensitive the option should be set to NO to properly deal with # output files written for symbols that only differ in casing, such as for two # classes, one named CLASS and the other named Class, and to also support # references to files without having to specify the exact matching casing. On # Windows (including Cygwin) and macOS, users should typically set this option # to NO, whereas on Linux or other Unix flavors it should typically be set to # YES. # Possible values are: SYSTEM, NO and YES. # The default value is: SYSTEM. CASE_SENSE_NAMES = SYSTEM # If the HIDE_SCOPE_NAMES tag is set to NO then Doxygen will show members with # their full class and namespace scopes in the documentation. If set to YES, the # scope will be hidden. # The default value is: NO. HIDE_SCOPE_NAMES = NO # If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then Doxygen will # append additional text to a page's title, such as Class Reference. If set to # YES the compound reference will be hidden. # The default value is: NO. HIDE_COMPOUND_REFERENCE= NO # If the SHOW_HEADERFILE tag is set to YES then the documentation for a class # will show which file needs to be included to use the class. # The default value is: YES. SHOW_HEADERFILE = YES # If the SHOW_INCLUDE_FILES tag is set to YES then Doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. SHOW_INCLUDE_FILES = YES # If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each # grouped member an include statement to the documentation, telling the reader # which file to include in order to use the member. # The default value is: NO. SHOW_GROUPED_MEMB_INC = NO # If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen will list include # files with double quotes in the documentation rather than with sharp brackets. # The default value is: NO. FORCE_LOCAL_INCLUDES = NO # If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the # documentation for inline members. # The default value is: YES. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES then Doxygen will sort the # (detailed) documentation of file and class members alphabetically by member # name. If set to NO, the members will appear in declaration order. # The default value is: YES. SORT_MEMBER_DOCS = YES # If the SORT_BRIEF_DOCS tag is set to YES then Doxygen will sort the brief # descriptions of file, namespace and class members alphabetically by member # name. If set to NO, the members will appear in declaration order. Note that # this will also influence the order of the classes in the class list. # The default value is: NO. SORT_BRIEF_DOCS = NO # If the SORT_MEMBERS_CTORS_1ST tag is set to YES then Doxygen will sort the # (brief and detailed) documentation of class members so that constructors and # destructors are listed first. If set to NO the constructors will appear in the # respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. # Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief # member documentation. # Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting # detailed member documentation. # The default value is: NO. SORT_MEMBERS_CTORS_1ST = NO # If the SORT_GROUP_NAMES tag is set to YES then Doxygen will sort the hierarchy # of group names into alphabetical order. If set to NO the group names will # appear in their defined order. # The default value is: NO. SORT_GROUP_NAMES = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by # fully-qualified names, including namespaces. If set to NO, the class list will # be sorted only by class name, not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the alphabetical # list. # The default value is: NO. SORT_BY_SCOPE_NAME = NO # If the STRICT_PROTO_MATCHING option is enabled and Doxygen fails to do proper # type resolution of all parameters of a function it will reject a match between # the prototype and the implementation of a member function even if there is # only one candidate or it is obvious which candidate to choose by doing a # simple string match. By disabling STRICT_PROTO_MATCHING Doxygen will still # accept a match between prototype and implementation in such cases. # The default value is: NO. STRICT_PROTO_MATCHING = NO # The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo # list. This list is created by putting \todo commands in the documentation. # The default value is: YES. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test # list. This list is created by putting \test commands in the documentation. # The default value is: YES. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug # list. This list is created by putting \bug commands in the documentation. # The default value is: YES. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) # the deprecated list. This list is created by putting \deprecated commands in # the documentation. # The default value is: YES. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional documentation # sections, marked by \if ... \endif and \cond # ... \endcond blocks. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the # initial value of a variable or macro / define can have for it to appear in the # documentation. If the initializer consists of more lines than specified here # it will be hidden. Use a value of 0 to hide initializers completely. The # appearance of the value of individual variables and macros / defines can be # controlled using \showinitializer or \hideinitializer command in the # documentation regardless of this setting. # Minimum value: 0, maximum value: 10000, default value: 30. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated at # the bottom of the documentation of classes and structs. If set to YES, the # list will mention the files that were used to generate the documentation. # The default value is: YES. SHOW_USED_FILES = YES # Set the SHOW_FILES tag to NO to disable the generation of the Files page. This # will remove the Files entry from the Quick Index and from the Folder Tree View # (if specified). # The default value is: YES. SHOW_FILES = YES # Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces # page. This will remove the Namespaces entry from the Quick Index and from the # Folder Tree View (if specified). # The default value is: YES. SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # Doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via # popen()) the command command input-file, where command is the value of the # FILE_VERSION_FILTER tag, and input-file is the name of an input file provided # by Doxygen. Whatever the program writes to standard output is used as the file # version. For an example see the documentation. FILE_VERSION_FILTER = # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed # by Doxygen. The layout file controls the global structure of the generated # output files in an output format independent way. To create the layout file # that represents Doxygen's defaults, run Doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml # will be used as the name of the layout file. See also section "Changing the # layout of pages" for information. # # Note that if you run Doxygen from a directory containing a file called # DoxygenLayout.xml, Doxygen will parse it automatically even if the LAYOUT_FILE # tag is left empty. LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files containing # the reference definitions. This must be a list of .bib files. The .bib # extension is automatically appended if omitted. This requires the bibtex tool # to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. # For LaTeX the style of the bibliography can be controlled using # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the # search path. See also \cite for info how to create references. CITE_BIB_FILES = # The EXTERNAL_TOOL_PATH tag can be used to extend the search path (PATH # environment variable) so that external tools such as latex and gs can be # found. # Note: Directories specified with EXTERNAL_TOOL_PATH are added in front of the # path already specified by the PATH variable, and are added in the order # specified. # Note: This option is particularly useful for macOS version 14 (Sonoma) and # higher, when running Doxygen from Doxywizard, because in this case any user- # defined changes to the PATH are ignored. A typical example on macOS is to set # EXTERNAL_TOOL_PATH = /Library/TeX/texbin /usr/local/bin # together with the standard path, the full search path used by doxygen when # launching external tools will then become # PATH=/Library/TeX/texbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin EXTERNAL_TOOL_PATH = #--------------------------------------------------------------------------- # Configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated to # standard output by Doxygen. If QUIET is set to YES this implies that the # messages are off. # The default value is: NO. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error (stderr) by Doxygen. If WARNINGS is set to YES # this implies that the warnings are on. # # Tip: Turn warnings on while writing the documentation. # The default value is: YES. WARNINGS = YES # If the WARN_IF_UNDOCUMENTED tag is set to YES then Doxygen will generate # warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag # will automatically be disabled. # The default value is: YES. WARN_IF_UNDOCUMENTED = NO # If the WARN_IF_DOC_ERROR tag is set to YES, Doxygen will generate warnings for # potential errors in the documentation, such as documenting some parameters in # a documented function twice, or documenting parameters that don't exist or # using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES # If WARN_IF_INCOMPLETE_DOC is set to YES, Doxygen will warn about incomplete # function parameter documentation. If set to NO, Doxygen will accept that some # parameters have no documentation without warning. # The default value is: YES. WARN_IF_INCOMPLETE_DOC = YES # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return # value. If set to NO, Doxygen will only warn about wrong parameter # documentation, but not about the absence of documentation. If EXTRACT_ALL is # set to YES then this flag will automatically be disabled. See also # WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = NO # If WARN_IF_UNDOC_ENUM_VAL option is set to YES, Doxygen will warn about # undocumented enumeration values. If set to NO, Doxygen will accept # undocumented enumeration values. If EXTRACT_ALL is set to YES then this flag # will automatically be disabled. # The default value is: NO. WARN_IF_UNDOC_ENUM_VAL = NO # If WARN_LAYOUT_FILE option is set to YES, Doxygen will warn about issues found # while parsing the user defined layout file, such as missing or wrong elements. # See also LAYOUT_FILE for details. If set to NO, problems with the layout file # will be suppressed. # The default value is: YES. WARN_LAYOUT_FILE = YES # If the WARN_AS_ERROR tag is set to YES then Doxygen will immediately stop when # a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS # then Doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but # at the end of the Doxygen process Doxygen will return with a non-zero status. # If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS_PRINT then Doxygen behaves # like FAIL_ON_WARNINGS but in case no WARN_LOGFILE is defined Doxygen will not # write the warning messages in between other messages but write them at the end # of a run, in case a WARN_LOGFILE is defined the warning messages will be # besides being in the defined file also be shown at the end of a run, unless # the WARN_LOGFILE is defined as - i.e. standard output (stdout) in that case # the behavior will remain as with the setting FAIL_ON_WARNINGS. # Possible values are: NO, YES, FAIL_ON_WARNINGS and FAIL_ON_WARNINGS_PRINT. # The default value is: NO. WARN_AS_ERROR = NO # The WARN_FORMAT tag determines the format of the warning messages that Doxygen # can produce. The string should contain the $file, $line, and $text tags, which # will be replaced by the file and line number from which the warning originated # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) # See also: WARN_LINE_FORMAT # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" # In the $text part of the WARN_FORMAT command it is possible that a reference # to a more specific place is given. To make it easier to jump to this place # (outside of Doxygen) the user can define a custom "cut" / "paste" string. # Example: # WARN_LINE_FORMAT = "'vi $file +$line'" # See also: WARN_FORMAT # The default value is: at line $line of file $file. WARN_LINE_FORMAT = "at line $line of file $file" # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard # error (stderr). In case the file specified cannot be opened for writing the # warning and error messages are written to standard error. When as file - is # specified the warning and error messages are written to standard output # (stdout). WARN_LOGFILE = #--------------------------------------------------------------------------- # Configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag is used to specify the files and/or directories that contain # documented source files. You may enter file names like myfile.cpp or # directories like /usr/src/myproject. Separate the files or directories with # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. INPUT = ../src/eckit # This tag can be used to specify the character encoding of the source files # that Doxygen parses. Internally Doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv # documentation (see: # https://www.gnu.org/software/libiconv/) for the list of possible encodings. # See also: INPUT_FILE_ENCODING # The default value is: UTF-8. INPUT_ENCODING = UTF-8 # This tag can be used to specify the character encoding of the source files # that Doxygen parses. The INPUT_FILE_ENCODING tag can be used to specify # character encoding on a per file pattern basis. Doxygen will compare the file # name with each pattern and apply the encoding instead of the default # INPUT_ENCODING if there is a match. The character encodings are a list of the # form: pattern=encoding (like *.php=ISO-8859-1). # See also: INPUT_ENCODING for further information on supported encodings. INPUT_FILE_ENCODING = # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. # # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # read by Doxygen. # # Note the list of default checked file patterns might differ from the list of # default file extension mappings. # # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cxxm, # *.cpp, *.cppm, *.ccm, *.c++, *.c++m, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, # *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, *.h++, *.ixx, *.l, *.cs, *.d, # *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to # be provided as Doxygen C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, # *.f18, *.f, *.for, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.c \ *.cc \ *.cxx \ *.cxxm \ *.cpp \ *.cppm \ *.ccm \ *.c++ \ *.c++m \ *.java \ *.ii \ *.ixx \ *.ipp \ *.i++ \ *.inl \ *.idl \ *.ddl \ *.odl \ *.h \ *.hh \ *.hxx \ *.hpp \ *.h++ \ *.ixx \ *.l \ *.cs \ *.d \ *.php \ *.php4 \ *.php5 \ *.phtml \ *.inc \ *.m \ *.markdown \ *.md \ *.mm \ *.dox \ *.py \ *.pyw \ *.f90 \ *.f95 \ *.f03 \ *.f08 \ *.f18 \ *.f \ *.for \ *.vhd \ *.vhdl \ *.ucf \ *.qsf \ *.ice # The RECURSIVE tag can be used to specify whether or not subdirectories should # be searched for input files as well. # The default value is: NO. RECURSIVE = YES # The EXCLUDE tag can be used to specify files and/or directories that should be # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. # # Note that relative paths are relative to the directory from which Doxygen is # run. EXCLUDE = # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded # from the input. # The default value is: NO. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. # # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* EXCLUDE_PATTERNS = "*/contrib/*" # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, # ANamespace::AClass, ANamespace::*Test EXCLUDE_SYMBOLS = # The EXAMPLE_PATH tag can be used to specify one or more files or directories # that contain example code fragments that are included (see the \include # command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and # *.h) to filter out the source-files in the directories. If left blank all # files are included. EXAMPLE_PATTERNS = * # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude commands # irrespective of the value of the RECURSIVE tag. # The default value is: NO. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or directories # that contain images that are to be included in the documentation (see the # \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that Doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command: # # # # where is the value of the INPUT_FILTER tag, and is the # name of an input file. Doxygen will then use the output that the filter # program writes to standard output. If FILTER_PATTERNS is specified, this tag # will be ignored. # # Note that the filter must not add or remove lines; it is applied before the # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. # # Note that Doxygen will use the data processed and written to standard output # for further processing, therefore nothing else, like debug statements or used # commands (so in case of a Windows batch file always use @echo OFF), should be # written to standard output. # # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # properly processed by Doxygen. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: pattern=filter # (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how # filters are used. If the FILTER_PATTERNS tag is empty or if none of the # patterns match the file name, INPUT_FILTER is applied. # # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # properly processed by Doxygen. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER) will also be used to filter the input files that are used for # producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). # The default value is: NO. FILTER_SOURCE_FILES = NO # The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file # pattern. A pattern will override the setting for FILTER_PATTERN (if any) and # it is also possible to disable source filtering for a specific pattern using # *.ext= (so without naming a filter). # This tag requires that the tag FILTER_SOURCE_FILES is set to YES. FILTER_SOURCE_PATTERNS = # If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that # is part of the input, its contents will be placed on the main page # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the Doxygen output. USE_MDFILE_AS_MAINPAGE = # If the IMPLICIT_DIR_DOCS tag is set to YES, any README.md file found in sub- # directories of the project's root, is used as the documentation for that sub- # directory, except when the README.md starts with a \dir, \page or \mainpage # command. If set to NO, the README.md file needs to start with an explicit \dir # command in order to be used as directory documentation. # The default value is: YES. IMPLICIT_DIR_DOCS = YES # The Fortran standard specifies that for fixed formatted Fortran code all # characters from position 72 are to be considered as comment. A common # extension is to allow longer lines before the automatic comment starts. The # setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can # be processed before the automatic comment starts. # Minimum value: 7, maximum value: 10000, default value: 72. FORTRAN_COMMENT_AFTER = 72 #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will be # generated. Documented entities will be cross-referenced with these sources. # # Note: To get rid of all source code in the generated output, make sure that # also VERBATIM_HEADERS is set to NO. # The default value is: NO. SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body of functions, # multi-line macros, enums or list initialized variables directly into the # documentation. # The default value is: NO. INLINE_SOURCES = NO # Setting the STRIP_CODE_COMMENTS tag to YES will instruct Doxygen to hide any # special comment blocks from generated source code fragments. Normal C, C++ and # Fortran comments will always remain visible. # The default value is: YES. STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES then for each documented # entity all documented functions referencing it will be listed. # The default value is: NO. REFERENCED_BY_RELATION = NO # If the REFERENCES_RELATION tag is set to YES then for each documented function # all documented entities called/used by that function will be listed. # The default value is: NO. REFERENCES_RELATION = NO # If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set # to YES then the hyperlinks from functions in REFERENCES_RELATION and # REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will # link to the documentation. # The default value is: YES. REFERENCES_LINK_SOURCE = YES # If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the # source code will show a tooltip with additional information such as prototype, # brief description and links to the definition and documentation. Since this # will make the HTML file larger and loading of large files a bit slower, you # can opt to disable this feature. # The default value is: YES. # This tag requires that the tag SOURCE_BROWSER is set to YES. SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will # point to the HTML generated by the htags(1) tool instead of Doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system # (see https://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. # # To use it do the following: # - Install the latest version of global # - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file # - Make sure the INPUT points to the root of the source tree # - Run doxygen as normal # # Doxygen will invoke htags (and that will in turn invoke gtags), so these # tools must be available from the command line (i.e. in the search path). # # The result: instead of the source browser generated by Doxygen, the links to # source code will now point to the output of htags. # The default value is: NO. # This tag requires that the tag SOURCE_BROWSER is set to YES. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set the YES then Doxygen will generate a # verbatim copy of the header file for each class for which an include is # specified. Set to NO to disable this. # See also: Section \class. # The default value is: YES. VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all # compounds will be generated. Enable this if the project contains a lot of # classes, structs, unions or interfaces. # The default value is: YES. ALPHABETICAL_INDEX = YES # The IGNORE_PREFIX tag can be used to specify a prefix (or a list of prefixes) # that should be ignored while generating the index headers. The IGNORE_PREFIX # tag works for classes, function and member names. The entity will be placed in # the alphabetical list under the first letter of the entity name that remains # after removing the prefix. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES, Doxygen will generate HTML output # The default value is: YES. GENERATE_HTML = NO # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of # it. # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). # The default value is: .html. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a user-defined HTML header file for # each generated HTML page. If the tag is left blank Doxygen will generate a # standard header. # # To get valid HTML the header file that includes any scripts and style sheets # that Doxygen needs, which is dependent on the configuration options used (e.g. # the setting GENERATE_TREEVIEW). It is highly recommended to start with a # default header using # doxygen -w html new_header.html new_footer.html new_stylesheet.css # YourConfigFile # and then modify the file new_header.html. See also section "Doxygen usage" # for information on how to generate the default header that Doxygen normally # uses. # Note: The header is subject to change so you typically have to regenerate the # default header when upgrading to a newer version of Doxygen. For a description # of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_HEADER = # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each # generated HTML page. If the tag is left blank Doxygen will generate a standard # footer. See HTML_HEADER for more information on how to generate a default # footer and what special commands can be used inside the footer. See also # section "Doxygen usage" for information on how to generate the default footer # that Doxygen normally uses. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading style # sheet that is used by each HTML page. It can be used to fine-tune the look of # the HTML output. If left blank Doxygen will generate a default style sheet. # See also section "Doxygen usage" for information on how to generate the style # sheet that Doxygen normally uses. # Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as # it is more robust and this tag (HTML_STYLESHEET) will in the future become # obsolete. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_STYLESHEET = # The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined # cascading style sheets that are included after the standard style sheets # created by Doxygen. Using this option one can overrule certain style aspects. # This is preferred over using HTML_STYLESHEET since it does not replace the # standard style sheet and is therefore more robust against future updates. # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra style sheet files is of importance (e.g. the last # style sheet in the list overrules the setting of the previous ones in the # list). # Note: Since the styling of scrollbars can currently not be overruled in # Webkit/Chromium, the styling will be left out of the default doxygen.css if # one or more extra stylesheets have been specified. So if scrollbar # customization is desired it has to be added explicitly. For an example see the # documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or # other source files which should be copied to the HTML output directory. Note # that these files will be copied to the base HTML output directory. Use the # $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these # files. In the HTML_STYLESHEET file, use the file name only. Also note that the # files will be copied as-is; there are no commands or markers available. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_FILES = # The HTML_COLORSTYLE tag can be used to specify if the generated HTML output # should be rendered with a dark or light theme. # Possible values are: LIGHT always generates light mode output, DARK always # generates dark mode output, AUTO_LIGHT automatically sets the mode according # to the user preference, uses light mode if no preference is set (the default), # AUTO_DARK automatically sets the mode according to the user preference, uses # dark mode if no preference is set and TOGGLE allows a user to switch between # light and dark mode via a button. # The default value is: AUTO_LIGHT. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE = AUTO_LIGHT # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to # this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. # Minimum value: 0, maximum value: 359, default value: 220. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors # in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_SAT = 100 # The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the # luminance component of the colors in the HTML output. Values below 100 # gradually make the output lighter, whereas values above 100 make the output # darker. The value divided by 100 is the actual gamma applied, so 80 represents # a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not # change the gamma. # Minimum value: 40, maximum value: 240, default value: 80. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COLORSTYLE_GAMMA = 80 # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML # documentation will contain a main index with vertical navigation menus that # are dynamically created via JavaScript. If disabled, the navigation index will # consists of multiple levels of tabs that are statically embedded in every HTML # page. Disable this option to support browsers that do not have JavaScript, # like the Qt help browser. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_DYNAMIC_MENUS = YES # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_DYNAMIC_SECTIONS = NO # If the HTML_CODE_FOLDING tag is set to YES then classes and functions can be # dynamically folded and expanded in the generated HTML source code. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_CODE_FOLDING = YES # If the HTML_COPY_CLIPBOARD tag is set to YES then Doxygen will show an icon in # the top right corner of code and text fragments that allows the user to copy # its content to the clipboard. Note this only works if supported by the browser # and the web page is served via a secure context (see: # https://www.w3.org/TR/secure-contexts/), i.e. using the https: or file: # protocol. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_COPY_CLIPBOARD = YES # Doxygen stores a couple of settings persistently in the browser (via e.g. # cookies). By default these settings apply to all HTML pages generated by # Doxygen across all projects. The HTML_PROJECT_COOKIE tag can be used to store # the settings under a project specific key, such that the user preferences will # be stored separately. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_PROJECT_COOKIE = # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to # such a level that at most the specified number of entries are visible (unless # a fully collapsed tree already exceeds this amount). So setting the number of # entries 1 will produce a full collapsed tree by default. 0 is a special value # representing an infinite number of entries and will result in a full expanded # tree by default. # Minimum value: 0, maximum value: 9999, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development # environment (see: # https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To # create a documentation set, Doxygen will generate a Makefile in the HTML # output directory. Running make will produce the docset in that directory and # running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy # genXcode/_index.html for more information. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_DOCSET = NO # This tag determines the name of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider # (such as a company or product suite) can be grouped. # The default value is: Doxygen generated docs. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_FEEDNAME = "Doxygen generated docs" # This tag determines the URL of the docset feed. A documentation feed provides # an umbrella under which multiple documentation sets from a single provider # (such as a company or product suite) can be grouped. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_FEEDURL = # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_BUNDLE_ID = org.doxygen.Project # The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style # string, e.g. com.mycompany.MyDocSet.documentation. # The default value is: org.doxygen.Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_ID = org.doxygen.Publisher # The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. # The default value is: Publisher. # This tag requires that the tag GENERATE_DOCSET is set to YES. DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then Doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop # on Windows. In the beginning of 2021 Microsoft took the original page, with # a.o. the download links, offline (the HTML help workshop was already many # years in maintenance mode). You can download the HTML help workshop from the # web archives at Installation executable (see: # http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo # ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by Doxygen into a single compiled HTML file (.chm). Compiled HTML # files are now used as the Windows 98 help format, and will replace the old # Windows help format (.hlp) on all Windows platforms in the future. Compressed # HTML files also contain an index, a table of contents, and you can search for # words in the documentation. The HTML workshop also contains a viewer for # compressed HTML files. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_HTMLHELP = NO # The CHM_FILE tag can be used to specify the file name of the resulting .chm # file. You can add a path in front of the file if the result should not be # written to the html output directory. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_FILE = # The HHC_LOCATION tag can be used to specify the location (absolute path # including file name) of the HTML help compiler (hhc.exe). If non-empty, # Doxygen will try to run the HTML help compiler on the generated index.hhp. # The file has to be specified with full path. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. HHC_LOCATION = # The GENERATE_CHI flag controls if a separate .chi index file is generated # (YES) or that it should be included in the main .chm file (NO). # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. GENERATE_CHI = NO # The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) # and project file content. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_INDEX_ENCODING = # The BINARY_TOC flag controls whether a binary table of contents is generated # (YES) or a normal table of contents (NO) in the .chm file. Furthermore it # enables the Previous and Next buttons. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members to # the table of contents of the HTML help documentation and to the tree view. # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. TOC_EXPAND = NO # The SITEMAP_URL tag is used to specify the full URL of the place where the # generated documentation will be placed on the server by the user during the # deployment of the documentation. The generated sitemap is called sitemap.xml # and placed on the directory specified by HTML_OUTPUT. In case no SITEMAP_URL # is specified no sitemap is generated. For information about the sitemap # protocol see https://www.sitemaps.org # This tag requires that the tag GENERATE_HTML is set to YES. SITEMAP_URL = # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help # (.qch) of the generated HTML documentation. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_QHP = NO # If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify # the file name of the resulting .qch file. The path specified is relative to # the HTML output folder. # This tag requires that the tag GENERATE_QHP is set to YES. QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace # (see: # https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual # Folders (see: # https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom # Filters (see: # https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom # Filters (see: # https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: # https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = # The QHG_LOCATION tag can be used to specify the location (absolute path # including file name) of Qt's qhelpgenerator. If non-empty Doxygen will try to # run qhelpgenerator on the generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = # If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be # generated, together with the HTML files, they form an Eclipse help plugin. To # install this plugin and make it available under the help contents menu in # Eclipse, the contents of the directory containing the HTML and XML files needs # to be copied into the plugins directory of eclipse. The name of the directory # within the plugins directory should be the same as the ECLIPSE_DOC_ID value. # After copying Eclipse needs to be restarted before the help appears. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_ECLIPSEHELP = NO # A unique identifier for the Eclipse help plugin. When installing the plugin # the directory name containing the HTML and XML files should also have this # name. Each documentation set should have its own identifier. # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. ECLIPSE_DOC_ID = org.doxygen.Project # If you want full control over the layout of the generated HTML pages it might # be necessary to disable the index and replace it with your own. The # DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top # of each HTML page. A value of NO enables the index and the value YES disables # it. Since the tabs in the index contain the same information as the navigation # tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. DISABLE_INDEX = YES # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag # value is set to YES, a side panel will be generated containing a tree-like # index structure (just like the one that is generated for HTML Help). For this # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can # further fine tune the look of the index (see "Fine-tuning the output"). As an # example, the default style sheet generated by Doxygen has an example that # shows how to put an image at the root of the tree instead of the PROJECT_NAME. # Since the tree basically has the same information as the tab index, you could # consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = YES # When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the # FULL_SIDEBAR option determines if the side bar is limited to only the treeview # area (value NO) or if it should extend to the full height of the window (value # YES). Setting this to YES gives a layout similar to # https://docs.readthedocs.io with more room for contents, but less room for the # project logo, title, and description. If either GENERATE_TREEVIEW or # DISABLE_INDEX is set to NO, this option has no effect. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. FULL_SIDEBAR = NO # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # Doxygen will group on one line in the generated HTML documentation. # # Note that a value of 0 will completely suppress the enum values from appearing # in the overview section. # Minimum value: 0, maximum value: 20, default value: 4. # This tag requires that the tag GENERATE_HTML is set to YES. ENUM_VALUES_PER_LINE = 4 # When the SHOW_ENUM_VALUES tag is set doxygen will show the specified # enumeration values besides the enumeration mnemonics. # The default value is: NO. SHOW_ENUM_VALUES = NO # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used # to set the initial width (in pixels) of the frame in which the tree is shown. # Minimum value: 0, maximum value: 1500, default value: 250. # This tag requires that the tag GENERATE_HTML is set to YES. TREEVIEW_WIDTH = 250 # If the EXT_LINKS_IN_WINDOW option is set to YES, Doxygen will open links to # external symbols imported via tag files in a separate window. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. EXT_LINKS_IN_WINDOW = NO # If the OBFUSCATE_EMAILS tag is set to YES, Doxygen will obfuscate email # addresses. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. OBFUSCATE_EMAILS = YES # If the HTML_FORMULA_FORMAT option is set to svg, Doxygen will use the pdf2svg # tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see # https://inkscape.org) to generate formulas as SVG images instead of PNGs for # the HTML output. These images will generally look nicer at scaled resolutions. # Possible values are: png (the default) and svg (looks nicer but requires the # pdf2svg or inkscape tool). # The default value is: png. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FORMULA_FORMAT = png # Use this tag to change the font size of LaTeX formulas included as images in # the HTML documentation. When you change the font size after a successful # Doxygen run you need to manually remove any form_*.png images from the HTML # output directory to force them to be regenerated. # Minimum value: 8, maximum value: 50, default value: 10. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_FONTSIZE = 10 # The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands # to create new LaTeX commands to be used in formulas as building blocks. See # the section "Including formulas" for details. FORMULA_MACROFILE = # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see # https://www.mathjax.org) which uses client side JavaScript for the rendering # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path # to it using the MATHJAX_RELPATH option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. USE_MATHJAX = NO # With MATHJAX_VERSION it is possible to specify the MathJax version to be used. # Note that the different versions of MathJax have different requirements with # regards to the different settings, so it is possible that also other MathJax # settings have to be changed when switching between the different MathJax # versions. # Possible values are: MathJax_2 and MathJax_3. # The default value is: MathJax_2. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_VERSION = MathJax_2 # When MathJax is enabled you can set the default output format to be used for # the MathJax output. For more details about the output format see MathJax # version 2 (see: # http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 # (see: # http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best # compatibility. This is the name for Mathjax version 2, for MathJax version 3 # this will be translated into chtml), NativeMML (i.e. MathML. Only supported # for MathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This # is the name for Mathjax version 3, for MathJax version 2 this will be # translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_FORMAT = HTML-CSS # When MathJax is enabled you need to specify the location relative to the HTML # output directory using the MATHJAX_RELPATH option. The destination directory # should contain the MathJax.js script. For instance, if the mathjax directory # is located at the same level as the HTML output directory, then # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of # MathJax from https://www.mathjax.org before deployment. The default value is: # - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 # - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example # for MathJax version 2 (see # https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols # For example for MathJax version 3 (see # http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): # MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with JavaScript pieces # of code that will be used on startup of the MathJax code. See the MathJax site # (see: # http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_CODEFILE = # When the SEARCHENGINE tag is enabled Doxygen will generate a search box for # the HTML output. The underlying search engine uses JavaScript and DHTML and # should work on any modern browser. Note that when using HTML help # (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) # there is already a search function so this one should typically be disabled. # For large projects the JavaScript based search engine can be slow, then # enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to # search using the keyboard; to jump to the search box use + S # (what the is depends on the OS and browser, but it is typically # , /