pax_global_header00006660000000000000000000000064151142543310014511gustar00rootroot0000000000000052 comment=33ec4b061ad0a6e5508e679e150ce6422ef6be30 liblognorm-2.0.8/000077500000000000000000000000001511425433100136645ustar00rootroot00000000000000liblognorm-2.0.8/.github/000077500000000000000000000000001511425433100152245ustar00rootroot00000000000000liblognorm-2.0.8/.github/workflows/000077500000000000000000000000001511425433100172615ustar00rootroot00000000000000liblognorm-2.0.8/.github/workflows/run_centos_7.yml000066400000000000000000000037341511425433100224200ustar00rootroot00000000000000# Copyright 2020 Rainer Gerhards and Others # # https://github.com/rsyslog/rsyslog-pkg-ubuntu # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # References: # # https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options # https://github.com/settings/notifications # https://software.opensuse.org//download.html?project=home%3Argerhards&package=rsyslog --- name: check centos 7 on: pull_request: jobs: CI: runs-on: ubuntu-18.04 timeout-minutes: 10 steps: - name: git checkout project uses: actions/checkout@v1 - name: run container CI pipeline run: | chmod -R go+rw . export RSYSLOG_CONTAINER_UID="" # use default export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php' export CFLAGS='-g' export CC='gcc' export USE_AUTO_DEBUG='off' export CI_MAKE_OPT='-j20' export CI_MAKE_CHECK_OPT='-j1' export CI_CHECK_CMD='check' export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_centos:7' # we need to override rsyslog configure options to avoid side-effects! export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE='' devtools/devcontainer.sh --rm devtools/run-ci.sh - name: show error logs (if we errored) if: ${{ failure() || cancelled() }} run: | devtools/gather-check-logs.sh cat failed-tests.log liblognorm-2.0.8/.github/workflows/run_centos_8.yml000066400000000000000000000037341511425433100224210ustar00rootroot00000000000000# Copyright 2020 Rainer Gerhards and Others # # https://github.com/rsyslog/rsyslog-pkg-ubuntu # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # References: # # https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options # https://github.com/settings/notifications # https://software.opensuse.org//download.html?project=home%3Argerhards&package=rsyslog --- name: check centos 8 on: pull_request: jobs: CI: runs-on: ubuntu-18.04 timeout-minutes: 10 steps: - name: git checkout project uses: actions/checkout@v1 - name: run container CI pipeline run: | chmod -R go+rw . export RSYSLOG_CONTAINER_UID="" # use default export RSYSLOG_STATSURL='http://build.rsyslog.com/testbench-failedtest.php' export CFLAGS='-g' export CC='gcc' export USE_AUTO_DEBUG='off' export CI_MAKE_OPT='-j20' export CI_MAKE_CHECK_OPT='-j1' export CI_CHECK_CMD='check' export RSYSLOG_DEV_CONTAINER='rsyslog/rsyslog_dev_base_centos:8' # we need to override rsyslog configure options to avoid side-effects! export RSYSLOG_CONFIGURE_OPTIONS_OVERRIDE='' devtools/devcontainer.sh --rm devtools/run-ci.sh - name: show error logs (if we errored) if: ${{ failure() || cancelled() }} run: | devtools/gather-check-logs.sh cat failed-tests.log liblognorm-2.0.8/.gitignore000066400000000000000000000006201511425433100156520ustar00rootroot00000000000000INSTALL Makefile Makefile.in aclocal.m4 autom4te.cache compile config.guess config.h config.h.in config.h.in~ config.log config.status config.sub configure depcomp install-sh libtool lognorm.pc ltmain.sh m4/libtool.m4 m4/ltoptions.m4 m4/ltsugar.m4 m4/ltversion.m4 m4/lt~obsolete.m4 missing stamp-h1 test-driver *.trs *.log test.out src/ln_test tests/options.sh compat/.deps compat/.libs *.la *.lo *.oliblognorm-2.0.8/.travis.yml000066400000000000000000000044421511425433100160010ustar00rootroot00000000000000#group: deprecated language: c compiler: - gcc - clang matrix: exclude: - compiler: gcc - compiler: clang include: - sudo: required compiler: gcc - sudo: required compiler: gcc dist: trusty # We disable 12.04 because we have a header issue here w/ clang #- sudo: required # compiler: clang - sudo: required compiler: clang dist: trusty before_install: - travis_retry sudo apt-get update -qq - travis_retry sudo apt-get install -qq libpcre3-dev libpcre3-dbg valgrind python-pip - travis_retry sudo pip install -U sphinx - travis_retry sudo add-apt-repository ppa:adiscon/v8-stable -y - travis_retry sudo apt-get update -qq install: - travis_retry sudo apt-get install -qq libestr-dev libfastjson-dev - travis_retry sudo apt-get install -qq clang # the following is a work-around to solve the # "too old autoconf-archive" problem - mkdir tmp - cd tmp - git clone git://git.sv.gnu.org/autoconf-archive.git - sudo cp autoconf-archive/m4/* /usr/share/aclocal - cd .. - rm -rf tmp script: - CI/check_codestyle.sh - autoreconf --force --verbose --install # note: we enable regexp to check the v1 compatibility layer. v2 does # not have it, nor is it recommended to enable it. - if [ "$CC" == "gcc" ] ; then ./configure --prefix=/opt/liblognorm --build=x86_64-pc-linux-gnu --host=x86_64-pc-linux-gnu --mandir=/usr/share/man --infodir=/usr/share/info --datadir=/usr/share --sysconfdir=/etc --localstatedir=/var/lib --disable-dependency-tracking --libdir=/usr/lib64 --enable-debug --enable-testbench --enable-docs --enable-regexp --enable-valgrind; fi - if [ "$CC" == "gcc" ] ; then make && make dist && make check && sudo make install; fi # here come the clang test. So far, we just call the static analyzer - if [ "$CC" == "clang" ] ; then export CFLAGS="-Werror -Wfatal-errors -std=c99" ; fi - if [ "$CC" == "clang" ] ; then scan-build ./configure --prefix=/opt/liblognorm --build=x86_64-pc-linux-gnu --host=x86_64-pc-linux-gnu --mandir=/usr/share/man --infodir=/usr/share/info --datadir=/usr/share --sysconfdir=/etc --localstatedir=/var/lib --disable-dependency-tracking --libdir=/usr/lib64 --enable-debug --enable-testbench --enable-regexp ; fi - if [ "$CC" == "clang" ] ; then scan-build --status-bugs make ; fi liblognorm-2.0.8/AUTHORS000066400000000000000000000000661511425433100147360ustar00rootroot00000000000000Rainer Gerhards , Adiscon GmbH liblognorm-2.0.8/CI/000077500000000000000000000000001511425433100141575ustar00rootroot00000000000000liblognorm-2.0.8/CI/check_codestyle.sh000077500000000000000000000004041511425433100176440ustar00rootroot00000000000000#!/bin/bash mkdir _tmp_stylecheck cd _tmp_stylecheck git clone https://github.com/rsyslog/codestyle cd codestyle gcc --std=c99 stylecheck.c -o stylecheck cd ../.. find . -name "*.[ch]" | xargs _tmp_stylecheck/codestyle/stylecheck -w -f -l 120 rm -rf codestyle liblognorm-2.0.8/COPYING000066400000000000000000000626201511425433100147250ustar00rootroot00000000000000 GNU LESSER GENERAL PUBLIC LICENSE Version 2.1, February 1999 Copyright (C) 1991, 1999 Free Software Foundation, Inc. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. [This is the first released version of the Lesser GPL. It also counts as the successor of the GNU Library Public License, version 2, hence the version number 2.1.] Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public Licenses are intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This license, the Lesser General Public License, applies to some specially designated software packages--typically libraries--of the Free Software Foundation and other authors who decide to use it. You can use it too, but we suggest you first think carefully about whether this license or the ordinary General Public License is the better strategy to use in any particular case, based on the explanations below. When we speak of free software, we are referring to freedom of use, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish); that you receive source code or can get it if you want it; that you can change the software and use pieces of it in new free programs; and that you are informed that you can do these things. To protect your rights, we need to make restrictions that forbid distributors to deny you these rights or to ask you to surrender these rights. These restrictions translate to certain responsibilities for you if you distribute copies of the library or if you modify it. For example, if you distribute copies of the library, whether gratis or for a fee, you must give the recipients all the rights that we gave you. You must make sure that they, too, receive or can get the source code. If you link other code with the library, you must provide complete object files to the recipients, so that they can relink them with the library after making changes to the library and recompiling it. And you must show them these terms so they know their rights. We protect your rights with a two-step method: (1) we copyright the library, and (2) we offer you this license, which gives you legal permission to copy, distribute and/or modify the library. To protect each distributor, we want to make it very clear that there is no warranty for the free library. Also, if the library is modified by someone else and passed on, the recipients should know that what they have is not the original version, so that the original author's reputation will not be affected by problems that might be introduced by others. Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot effectively restrict the users of a free program by obtaining a restrictive license from a patent holder. Therefore, we insist that any patent license obtained for a version of the library must be consistent with the full freedom of use specified in this license. Most GNU software, including some libraries, is covered by the ordinary GNU General Public License. This license, the GNU Lesser General Public License, applies to certain designated libraries, and is quite different from the ordinary General Public License. We use this license for certain libraries in order to permit linking those libraries into non-free programs. When a program is linked with a library, whether statically or using a shared library, the combination of the two is legally speaking a combined work, a derivative of the original library. The ordinary General Public License therefore permits such linking only if the entire combination fits its criteria of freedom. The Lesser General Public License permits more lax criteria for linking other code with the library. We call this license the "Lesser" General Public License because it does Less to protect the user's freedom than the ordinary General Public License. It also provides other free software developers Less of an advantage over competing non-free programs. These disadvantages are the reason we use the ordinary General Public License for many libraries. However, the Lesser license provides advantages in certain special circumstances. For example, on rare occasions, there may be a special need to encourage the widest possible use of a certain library, so that it becomes a de-facto standard. To achieve this, non-free programs must be allowed to use the library. A more frequent case is that a free library does the same job as widely used non-free libraries. In this case, there is little to gain by limiting the free library to free software only, so we use the Lesser General Public License. In other cases, permission to use a particular library in non-free programs enables a greater number of people to use a large body of free software. For example, permission to use the GNU C Library in non-free programs enables many more people to use the whole GNU operating system, as well as its variant, the GNU/Linux operating system. Although the Lesser General Public License is Less protective of the users' freedom, it does ensure that the user of a program that is linked with the Library has the freedom and the wherewithal to run that program using a modified version of the Library. The precise terms and conditions for copying, distribution and modification follow. Pay close attention to the difference between a "work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must be combined with the library in order to run. GNU LESSER GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License Agreement applies to any software library or other program which contains a notice placed by the copyright holder or other authorized party saying it may be distributed under the terms of this Lesser General Public License (also called "this License"). Each licensee is addressed as "you". A "library" means a collection of software functions and/or data prepared so as to be conveniently linked with application programs (which use some of those functions and data) to form executables. The "Library", below, refers to any such software library or work which has been distributed under these terms. A "work based on the Library" means either the Library or any derivative work under copyright law: that is to say, a work containing the Library or a portion of it, either verbatim or with modifications and/or translated straightforwardly into another language. (Hereinafter, translation is included without limitation in the term "modification".) "Source code" for a work means the preferred form of the work for making modifications to it. For a library, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the library. Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running a program using the Library is not restricted, and output from such a program is covered only if its contents constitute a work based on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does and what the program that uses the Library does. 1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and distribute a copy of this License along with the Library. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Library or any portion of it, thus forming a work based on the Library, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) The modified work must itself be a software library. b) You must cause the files modified to carry prominent notices stating that you changed the files and the date of any change. c) You must cause the whole of the work to be licensed at no charge to all third parties under the terms of this License. d) If a facility in the modified Library refers to a function or a table of data to be supplied by an application program that uses the facility, other than as an argument passed when the facility is invoked, then you must make a good faith effort to ensure that, in the event an application does not supply such function or table, the facility still operates, and performs whatever part of its purpose remains meaningful. (For example, a function in a library to compute square roots has a purpose that is entirely well-defined independent of the application. Therefore, Subsection 2d requires that any application-supplied function or table used by this function must be optional: if the application does not supply it, the square root function must still compute square roots.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Library, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Library, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Library. In addition, mere aggregation of another work not based on the Library with the Library (or with a work based on the Library) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may opt to apply the terms of the ordinary GNU General Public License instead of this License to a given copy of the Library. To do this, you must alter all the notices that refer to this License, so that they refer to the ordinary GNU General Public License, version 2, instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. This option is useful when you wish to copy part of the code of the Library into a program that is not a library. 4. You may copy and distribute the Library (or a portion or derivative of it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange. If distribution of object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place satisfies the requirement to distribute the source code, even though third parties are not compelled to copy the source along with the object code. 5. A program that contains no derivative of any portion of the Library, but is designed to work with the Library by being compiled or linked with it, is called a "work that uses the Library". Such a work, in isolation, is not a derivative work of the Library, and therefore falls outside the scope of this License. However, linking a "work that uses the Library" with the Library creates an executable that is a derivative of the Library (because it contains portions of the Library), rather than a "work that uses the library". The executable is therefore covered by this License. Section 6 states terms for distribution of such executables. When a "work that uses the Library" uses material from a header file that is part of the Library, the object code for the work may be a derivative work of the Library even though the source code is not. Whether this is true is especially significant if the work can be linked without the Library, or if the work is itself a library. The threshold for this to be true is not precisely defined by law. If such an object file uses only numerical parameters, data structure layouts and accessors, and small macros and small inline functions (ten lines or less in length), then the use of the object file is unrestricted, regardless of whether it is legally a derivative work. (Executables containing this object code plus portions of the Library will still fall under Section 6.) Otherwise, if the work is a derivative of the Library, you may distribute the object code for the work under the terms of Section 6. Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. 6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a work containing portions of the Library, and distribute that work under terms of your choice, provided that the terms permit modification of the work for the customer's own use and reverse engineering for debugging such modifications. You must give prominent notice with each copy of the work that the Library is used in it and that the Library and its use are covered by this License. You must supply a copy of this License. If the work during execution displays copyright notices, you must include the copyright notice for the Library among them, as well as a reference directing the user to the copy of this License. Also, you must do one of these things: a) Accompany the work with the complete corresponding machine-readable source code for the Library including whatever changes were used in the work (which must be distributed under Sections 1 and 2 above); and, if the work is an executable linked with the Library, with the complete machine-readable "work that uses the Library", as object code and/or source code, so that the user can modify the Library and then relink to produce a modified executable containing the modified Library. (It is understood that the user who changes the contents of definitions files in the Library will not necessarily be able to recompile the application to use the modified definitions.) b) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (1) uses at run time a copy of the library already present on the user's computer system, rather than copying library functions into the executable, and (2) will operate properly with a modified version of the library, if the user installs one, as long as the modified version is interface-compatible with the version that the work was made with. c) Accompany the work with a written offer, valid for at least three years, to give the same user the materials specified in Subsection 6a, above, for a charge no more than the cost of performing this distribution. d) If distribution of the work is made by offering access to copy from a designated place, offer equivalent access to copy the above specified materials from the same place. e) Verify that the user has already received a copy of these materials or that you have already sent this user a copy. For an executable, the required form of the "work that uses the Library" must include any data and utility programs needed for reproducing the executable from it. However, as a special exception, the materials to be distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. It may happen that this requirement contradicts the license restrictions of other proprietary libraries that do not normally accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined library, provided that the separate distribution of the work based on the Library and of the other library facilities is otherwise permitted, and provided that you do these two things: a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities. This must be distributed under the terms of the Sections above. b) Give prominent notice with the combined library of the fact that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. 8. You may not copy, modify, sublicense, link with, or distribute the Library except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense, link with, or distribute the Library is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 9. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Library or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Library (or any work based on the Library), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Library or works based on it. 10. Each time you redistribute the Library (or any work based on the Library), the recipient automatically receives a license from the original licensor to copy, distribute, link with or modify the Library subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with this License. 11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Library at all. For example, if a patent license would not permit royalty-free redistribution of the Library by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Library. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply, and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 12. If the distribution and/or use of the Library is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Library under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 13. The Free Software Foundation may publish revised and/or new versions of the Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Library specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Libraries If you develop a new library, and you want it to be of the greatest possible use to the public, we recommend making it free software that everyone can redistribute and change. You can do so by permitting redistribution under these terms (or, alternatively, under the terms of the ordinary General Public License). To apply these terms, attach the following notices to the library. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. liblognorm, a fast samples-based log normalization library Copyright (C) 2010 Rainer Gerhards This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA liblognorm-2.0.8/COPYING.ASL20000066400000000000000000000216611511425433100155050ustar00rootroot00000000000000Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 1. You must give any other recipients of the Work or Derivative Works a copy of this License; and 2. You must cause any modified files to carry prominent notices stating that You changed the files; and 3. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 4. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. liblognorm-2.0.8/ChangeLog000066400000000000000000000370471511425433100154510ustar00rootroot00000000000000---------------------------------------------------------------------- Version 2.0.8, 2025-12-04 - fix potential segfault on some platforms Thanks to Julian Thomas for a fix - fix memory leak when a custom type in rules does not match Thanks to Meric Sentunali for the fix and Julian Thomas for alerting me of the missing merge. ---------------------------------------------------------------------- Version 2.0.7, 2025-10-14 - parser(name-value-list): add separator character option adds new option "separator" for custom key/value pair separation (replaces whitespace when defined) - parser(name-value-list): add assignator option adds new option "assignator" for custom key/value assignment character (replaces '=' when defined, disables key name validation) - parser(name-value-list): add support for quoting and escaped characters values can now be quoted and include escaped characters such as backslash, separator, or double-quote - parser(name-value-list): fix parsing of escaped characters fixes incorrect handling of backslash, double-quote, and separator characters in quoted values - parser(name-value-list): add test for quoted values - parser(name-value-list): improve handling of invalid pairs stops parsing and sets offset if name/value is not valid, so remaining text can still be parsed - parser(CEF): fix empty last value handling if the last field is empty, it is now set to an empty string instead of failing - parser(CEF): fix header and escape sequence handling improved handling of slashes and trailing spaces in header parsing - CI: add GitHub Actions workflow for CI runs - code cleanup: removed problematic language from code base - maintenance: fixed various misspellings in code and comments - string rulebase bugfix: segfault when using LF in json rule If a json rule used a LF inside a string rule (one not loaded rulebase file), liblognorm segfaults. closes https://github.com/rsyslog/liblognorm/issues/333 ---------------------------------------------------------------------- Version 2.0.6, 2018-11-06 - implement Checkpoint LEA transfer format ... at least if we guess right that this is the format name. This type of format seems to be seen in syslog message. Checkpoint does not provide a spec, so everything is guesswork... :-( closes https://github.com/rsyslog/liblognorm/issues/309 - made build on AIX Thanks to Philippe Duveau for the patch. - fixes and improvements in bash scripting mostly based on shellcheck recommandations (via CodeFactor.com) - string parser: add "lazy" matching mode This introduces paramter "matching.lazy". See doc for details. - bugfix: suppress invalid param error for field name "-" Suppress invalid param error for name for hexnumber, float, number, date-rfc3164 and date-rfc5424. It will just check if name is "-" to make sure that we only suppress the error message in case we do not want to capture something. Thanks to Sol Huebner for the patch. closes https://github.com/rsyslog/liblognorm/issues/270 - bugfix: cisco-interface-spec did not succeed when at end of line Thanks to Sol Huebner for the patch. closes https://github.com/rsyslog/liblognorm/issues/229 ---------------------------------------------------------------------- Version 2.0.5, 2018-04-26 - bugfix: es_str2cstr leak in string-to v1 parser Thanks to Harshvardhan Shrivastava for the patch. - make "make check" "succeed" on solaris 10 actually, we just ignore the CI failures so that OpenCSW can build new packages. The problems actually exist on that platform, but testing has shown they always existed. We currently run out of time to really fixing this, plus we never had any bug report on Solaris (I assme nobody uses it on Solaris 10). However, that issues is a blocker to make new rsyslog versions available on OpenCSW for Solaris 10, so we go the dirty way of pretenting there is no problem. Note: the issues was orignally not seen, as the failing tests have been added later on. So the problem was always there, just not visible. - some mostly cosmetic fixes detected by Coverity Scan e. g. memory leak if and only if system was completely out of memory ---------------------------------------------------------------------- Version 2.0.4, 2017-10-04 - added support for native JSON number formats supported by parsers: number, float, hex - added support for creating unix timestamps supported by parsers: date-rfc3164, date-rfc5424 - fixed build problems on Solaris ... but there still seem to be some code issues, manifested in testbench failures. So use with care! ---------------------------------------------------------------------- Version 2.0.3, 2017-03-22 - add ability to load rulebase from a string introduces new API: int ln_loadSamplesFromString(ln_ctx ctx, const char *string); closes https://github.com/rsyslog/liblognorm/issues/239 - bugfix: string parser did not correctly parse word at end of line - bugfix: literal parser does not always store value if name is specified if rule=:%{"type":"literal", "text":"a", "name":"var"}% is used and matching message is provided, variable var ist not persisted. see also http://lists.adiscon.net/pipermail/rsyslog/2016-December/043985.html ---------------------------------------------------------------------- Version 2.0.2, 2016-11-15 - bugfix: no error was emitted on invalid "annotate" line - "annnotate": permit inline comments - fix a problem with cross-compilation see also: https://github.com/rsyslog/liblognorm/pull/221 Thanks to Luca Boccassi for the patch - testbench: add test for "annotate" functionality - bugfix: abort in literal path compaction when useing "alternative" parser When using the "alternative" parser, literals nodes could be created with multiple reference count. This is valid. However, literal path compaction did not consider this case, and so "merged" these nodes, which lead to pdag corruption and quickly to segfault. closes https://github.com/rsyslog/liblognorm/issues/220 closes https://github.com/rsyslog/liblognorm/issues/153 - bugfix: lognormalizer could loop This also caused the testbench to fail on some platforms. due too incorrect data type Thanks to Michael Biebl for this fix. - fix misleading compiler warning Thanks to Michael Biebl for this fix. - testbench: add test for "annotate" functionality ---------------------------------------------------------------------- Version 2.0.1, 2016-08-01 - fix public headers, which invalidly contained a strndup() definition Thanks to Michael Biebl for this fix. - fix some issues in pkgconfig file Thanks to Michael Biebl for this fix. - enhance build system to natively support systems with older autoconf versions and/or missing autoconf-archive. In this case we gracefully degrade functionality, but the build still is possible. Among others, this enables builds on CentOS 5. ---------------------------------------------------------------------- Version 2.0.0, 2016-07-21 - completely rewritten, much feature-enhanced version - requires libfastjson instead of json-c - big improvements to testbench runs, especially on travis among others, the static analyzer is now run and testbench throws an error if the static analyzer (via clang) is not clean - lognormalizer tool can now handle lines larger 10k characters Thanks to Janmejay Singh for the patch ---------------------------------------------------------------------- Version 1.1.3, 2015-??-?? [no official release] - make work on Solaris - check for runaway rules. A runaway rule is one that has unmatched percent signs and thus is not terminated properly at its end. This also means we no longer accept "rule=" at the first column of a continuation line, which is no problem (see doc for more information). - fix: process last line if it misses the terminating LF This problem occurs with the very last line of a rulebase (at EOF). If it is not properly terminated (LF missing), it is silently ignored. Previous versions did obviously process lines in this case. While technically this is invalid input, we can't outrule that such rulebases exist. For example, they do in the rsyslog testbench, which made us aware of the problem (see https://github.com/rsyslog/rsyslog/issues/489 ) I think the proper way of addressing this is to process such lines without termination, as many other tools do as well. closes https://github.com/rsyslog/liblognorm/issues/135 ---------------------------------------------------------------------- Version 1.1.2, 2015-07-20 - permit newline inside parser definition - new parser "cisco-interface-spec" - new parser "json" to process json parts of the message - new parser "mac48" to process mac layer addresses - new parser "name-value-list" (currently inofficial, experimental) - some parsers did incorrectly report success when an error occurred this was caused by inconsistencies between various macros. We have changed the parser-generation macros to match the semantics of the broader CHKN/CHKR macros and also restructured/simplified the parser generation macros. closes https://github.com/rsyslog/liblognorm/issues/41 - call "rest" parser only if nothing else matches. Versions prior to 1.1.2 did execute "rest" during regular parser processing, and thus parser matches have been more or less random. With 1.1.2 this is now always the last parser called. This may cause problems with existing rulesets, HOWEVER, adding any other rule or changing the load order would also have caused problems, so there really is no compatibility to preserve. see also: https://rainer.gerhards.net/2015/04/liblognorms-rest-parser-now-more-useful.html - new API to support error callbacks This permits callers to forward messages in regard to e.g. wrong rule bases to their users, which is very useful and actually missing in the previous code base. So far, we only have few error messages. However, we will review the code and add more. The important part is that callers can begin to use the new API and thus will benefit when we add more error messages. - testbench is now enabled by default - bugfix: misadressing on some constant values see also https://github.com/rsyslog/liblognorm/pull/67 Thanks to github user ontholerian for the patch - bugfix: add missing function prototypes This could potentially lead to problems on some platforms, especially those with 64 bit pointers. ---------------------------------------------------------------------- Version 1.1.1, 2015-03-09 - fixed library version numbering Thanks to Tomas Heinreich for reporting the problem. - added new parser syntaxes Thanks to Janmejay Singh for implementing most of them. - bugfix: function ln_parseFieldDescr() returns state value due to unitialized variable. This can also lead to invalid returning no sample node where one would have to be created. ---------------------------------------------------------------------- Version 1.1.0, 2015-01-08 - added regular expression support use this feature with great care, as it thrashes performance Thanks to Janmejay Singh for implementing this feature. - fix build problem when --enable-debug was set closes: https://github.com/rsyslog/liblognorm/issues/5 ---------------------------------------------------------------------- Version 1.0.1, 2014-04-11 - improved doc (via RST/Sphinx) - bugfix: unparsed fields were copied incorrectly from non-terminated string. Thanks to Josh Blum for the fix. - bugfix: mandatory tag did not work in lognormalizer ---------------------------------------------------------------------- Version 1.0.0, 2013-11-28 - WARNING: this version has incompatible interface and older programs will not compile with it. For details see http://www.liblognorm.com/news/on-liblognorm-1-0-0/ - libestr is not used any more in interface functions. Traditional C strings are used instead. Internally, libestr is still used, but scheduled for removal. - libee is not used any more. JSON-C is used for object handling instead. Parsers and formatters are now part of liblognorm. - added new field type "rest", which simply sinks all up to end of the string. - added support for glueing two fields together, without literal between them. It allows for constructs like: %volume:number%%unit:word% which matches string "1000Kbps" - Fix incorrect merging of trees with empty literal at end Thanks to Pavel Levshin for the patch - this version has survived many bugfixes ---------------------------------------------------------------------- ================================================================================ The versions below is liblognorm0, which has a different API ================================================================================ ---------------------------------------------------------------------- Version 0.3.7, 2013-07-17 - added support to load single samples Thanks to John Hopper for the patch ---------------------------------------------------------------------- Version 0.3.6, 2013-03-22 - bugfix: unitialized variable could lead to rulebase load error ---------------------------------------------------------------------- Version 0.3.5 (rgerhards), 2012-09-18 - renamed "normalizer" tool to "lognormalizer" to solve name clashes Thanks to the Fedora folks for pointing this out. ---------------------------------------------------------------------- Version 0.3.4 (rgerhards), 2012-04-16 - bugfix: normalizer tool had a memory leak Thanks to Brian Know for alerting me and helping to debug ---------------------------------------------------------------------- Version 0.3.3 (rgerhards), 2012-02-06 - required header file was not installed, resulting in compile error closes: http://bugzilla.adiscon.com/show_bug.cgi?id=307 Thanks to Andreis Vinogradovs for alerting us on this bug. ---------------------------------------------------------------------- Version 0.3.2 (rgerhards), 2011-11-21 - added rfc5424 parser (requires libee >= 0.3.2) - added "-" to serve as name for filler fields. Value is extracted, but no field is written - special handling for iptables log via %iptables% parser added (currently experimental pending practical verification) - normalizer tool on its way to a full-blow stand-alone tool - support for annotations added, for the time being see https://rainer.gerhards.net/2011/11/log-annotation-with-liblognorm.html ---------------------------------------------------------------------- Version 0.3.1 (rgerhards), 2011-04-18 - added -t option to normalizer so that only messages with a specified tag will be output - bugfix: abort if a tag was assigned to a message without any fields parsed out (uncommon scenario) - bugfix: mem leak on parse tree destruct -- associated tags were not deleted - bugfix: potential abort in normalizer due to misadressing in debug message generation ---------------------------------------------------------------------- Version 0.3.0 (rgerhards), 2011-04-06 - support for message classification via tags added - bugfix: partial messages were invalidly matched closes: http://bugzilla.adiscon.com/show_bug.cgi?id=247 ---------------------------------------------------------------------- Version 0.2.0 (rgerhards), 2011-04-01 - added -E option to normalizer tool, permits to specify data for encoders - support for new libee parsers: * Time12hr * Time24hr * ISODate * QuotedString - support for csv encoding added - added -p option to normalizer tool (output only correctly parsed entries) - bugfix: segfault if a parse tree prefix had exactly buffer size, in which case it was invalidly assumed that an external buffer had been allocated ---------------------------------------------------------------------- Version 0.1.0 (rgerhards), 2010-12-09 Initial public release. liblognorm-2.0.8/Doxyfile000066400000000000000000002110251511425433100153730ustar00rootroot00000000000000# Doxyfile 1.7.1 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project # # All text after a hash (#) is considered a comment and will be ignored # The format is: # TAG = value [value, ...] # For lists items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (" ") #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # This tag specifies the encoding used for all characters in the config file # that follow. The default is UTF-8 which is also the encoding used for all # text before the first occurrence of this tag. Doxygen uses libiconv (or the # iconv built into libc) for the transcoding. See # http://www.gnu.org/software/libiconv for the list of possible encodings. DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded # by quotes) that should identify the project. PROJECT_NAME = liblognorm # The PROJECT_NUMBER tag can be used to enter a project or revision number. # This could be handy for archiving the generated documentation or # if some version control system is used. PROJECT_NUMBER = 1.0.0 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) # base path where the generated documentation will be put. # If a relative path is entered, it will be relative to the location # where doxygen was started. If left blank the current directory will be used. OUTPUT_DIRECTORY = /home/rger/proj/liblognorm/doc # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create # 4096 sub-directories (in 2 levels) under the output directory of each output # format and will distribute the generated files over these directories. # Enabling this option can be useful when feeding doxygen a huge amount of # source files, where putting all generated files in the same directory would # otherwise cause performance problems for the file system. CREATE_SUBDIRS = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # The default language is English, other supported languages are: # Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, # Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, # Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English # messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, # Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak, # Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. OUTPUT_LANGUAGE = English # If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will # include brief member descriptions after the members that are listed in # the file and class documentation (similar to JavaDoc). # Set to NO to disable this. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend # the brief description of a member or function before the detailed description. # Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. REPEAT_BRIEF = YES # This tag implements a quasi-intelligent brief description abbreviator # that is used to form the text in various listings. Each string # in this list, if found as the leading text of the brief description, will be # stripped from the text and the result after processing the whole list, is # used as the annotated text. Otherwise, the brief description is used as-is. # If left blank, the following values are used ("$name" is automatically # replaced with the name of the entity): "The $name class" "The $name widget" # "The $name file" "is" "provides" "specifies" "contains" # "represents" "a" "an" "the" ABBREVIATE_BRIEF = "The $name class" \ "The $name widget" \ "The $name file" \ is \ provides \ specifies \ contains \ represents \ a \ an \ the # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # Doxygen will generate a detailed section even if there is only a brief # description. ALWAYS_DETAILED_SEC = NO # If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full # path before files name in the file list and in the header files. If set # to NO the shortest path that makes the file name unique will be used. FULL_PATH_NAMES = YES # If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag # can be used to strip a user-defined part of the path. Stripping is # only done if one of the specified strings matches the left-hand part of # the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the # path to strip. STRIP_FROM_PATH = # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of # the path mentioned in the documentation of a class, which tells # the reader which header file to include in order to use a class. # If left blank only the name of the header file containing the class # definition is used. Otherwise one should specify the include paths that # are normally passed to the compiler using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter # (but less readable) file names. This can be useful is your file systems # doesn't support long names like on DOS, Mac, or CD-ROM. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen # will interpret the first line (until the first dot) of a JavaDoc-style # comment as the brief description. If set to NO, the JavaDoc # comments will behave just like regular Qt-style comments # (thus requiring an explicit @brief command for a brief description.) JAVADOC_AUTOBRIEF = NO # If the QT_AUTOBRIEF tag is set to YES then Doxygen will # interpret the first line (until the first dot) of a Qt-style # comment as the brief description. If set to NO, the comments # will behave just like regular Qt-style comments (thus requiring # an explicit \brief command for a brief description.) QT_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen # treat a multi-line C++ special comment block (i.e. a block of //! or /// # comments) as a brief description. This used to be the default behaviour. # The new default is to treat a multi-line C++ comment block as a detailed # description. Set this tag to YES if you prefer the old behaviour instead. MULTILINE_CPP_IS_BRIEF = NO # If the INHERIT_DOCS tag is set to YES (the default) then an undocumented # member inherits the documentation from any documented member that it # re-implements. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce # a new page for each member. If set to NO, the documentation of a member will # be part of the file/class/namespace that contains it. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. # Doxygen uses this value to replace tabs by spaces in code fragments. TAB_SIZE = 8 # This tag can be used to specify a number of aliases that acts # as commands in the documentation. An alias has the form "name=value". # For example adding "sideeffect=\par Side Effects:\n" will allow you to # put the command \sideeffect (or @sideeffect) in the documentation, which # will result in a user-defined paragraph with heading "Side Effects:". # You can put \n's in the value part of an alias to insert newlines. ALIASES = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C # sources only. Doxygen will then generate output that is more tailored for C. # For instance, some of the names that are used will be different. The list # of all members will be omitted, etc. OPTIMIZE_OUTPUT_FOR_C = YES # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java # sources only. Doxygen will then generate output that is more tailored for # Java. For instance, namespaces will be presented as packages, qualified # scopes will look different, etc. OPTIMIZE_OUTPUT_JAVA = NO # Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran # sources only. Doxygen will then generate output that is more tailored for # Fortran. OPTIMIZE_FOR_FORTRAN = NO # Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL # sources. Doxygen will then generate output that is tailored for # VHDL. OPTIMIZE_OUTPUT_VHDL = NO # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given extension. # Doxygen has a built-in mapping, but you can override or extend it using this # tag. The format is ext=language, where ext is a file extension, and language # is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, # C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make # doxygen treat .inc files as Fortran files (default is PHP), and .f files as C # (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions # you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. EXTENSION_MAPPING = # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want # to include (a tag file for) the STL sources as input, then you should # set this tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); v.s. # func(std::string) {}). This also make the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. BUILTIN_STL_SUPPORT = NO # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. # Doxygen will parse them like normal C++ but will assume all classes use public # instead of private inheritance when no explicit protection keyword is present. SIP_SUPPORT = NO # For Microsoft's IDL there are propget and propput attributes to indicate getter # and setter methods for a property. Setting this option to YES (the default) # will make doxygen to replace the get and set methods by a property in the # documentation. This will only work if the methods are indeed getting or # setting a simple type. If this is not the case, or you want to show the # methods anyway, you should set this option to NO. IDL_PROPERTY_SUPPORT = YES # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES, then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. DISTRIBUTE_GROUP_DOC = NO # Set the SUBGROUPING tag to YES (the default) to allow class member groups of # the same type (for instance a group of public functions) to be put as a # subgroup of that type (e.g. under the Public Functions section). Set it to # NO to prevent subgrouping. Alternatively, this can be done per class using # the \nosubgrouping command. SUBGROUPING = YES # When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum # is documented as struct, union, or enum with the name of the typedef. So # typedef struct TypeS {} TypeT, will appear in the documentation as a struct # with name TypeT. When disabled the typedef will appear as a member of a file, # namespace, or class. And the struct will be named TypeS. This can typically # be useful for C code in case the coding convention dictates that all compound # types are typedef'ed and only the typedef is referenced, never the tag name. TYPEDEF_HIDES_STRUCT = NO # The SYMBOL_CACHE_SIZE determines the size of the internal cache use to # determine which symbols to keep in memory and which to flush to disk. # When the cache is full, less often used symbols will be written to disk. # For small to medium size projects (<1000 input files) the default value is # probably good enough. For larger projects a too small cache size can cause # doxygen to be busy swapping symbols to and from disk most of the time # causing a significant performance penality. # If the system has enough physical memory increasing the cache will improve the # performance by keeping more symbols in memory. Note that the value works on # a logarithmic scale so increasing the size by one will rougly double the # memory usage. The cache size is given by this formula: # 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, # corresponding to a cache size of 2^16 = 65536 symbols SYMBOL_CACHE_SIZE = 0 #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in # documentation are documented, even if no documentation was available. # Private class members and static file members will be hidden unless # the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES EXTRACT_ALL = NO # If the EXTRACT_PRIVATE tag is set to YES all private members of a class # will be included in the documentation. EXTRACT_PRIVATE = NO # If the EXTRACT_STATIC tag is set to YES all static members of a file # will be included in the documentation. EXTRACT_STATIC = NO # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) # defined locally in source files will be included in the documentation. # If set to NO only classes defined in header files are included. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. When set to YES local # methods, which are defined in the implementation section but not in # the interface are included in the documentation. # If set to NO (the default) only methods in the interface are included. EXTRACT_LOCAL_METHODS = NO # If this flag is set to YES, the members of anonymous namespaces will be # extracted and appear in the documentation as a namespace called # 'anonymous_namespace{file}', where file will be replaced with the base # name of the file that contains the anonymous namespace. By default # anonymous namespace are hidden. EXTRACT_ANON_NSPACES = NO # If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all # undocumented members of documented classes, files or namespaces. # If set to NO (the default) these members will be included in the # various overviews, but no documentation section is generated. # This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. # If set to NO (the default) these classes will be included in the various # overviews. This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all # friend (class|struct|union) declarations. # If set to NO (the default) these declarations will be included in the # documentation. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any # documentation blocks found inside the body of a function. # If set to NO (the default) these blocks will be appended to the # function's detailed documentation block. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation # that is typed after a \internal command is included. If the tag is set # to NO (the default) then the documentation will be excluded. # Set it to YES to include the internal documentation. INTERNAL_DOCS = NO # If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate # file names in lower-case letters. If set to YES upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # and Mac users are advised to set this option to NO. CASE_SENSE_NAMES = NO # If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen # will show members with their full class and namespace scopes in the # documentation. If set to YES the scope will be hidden. HIDE_SCOPE_NAMES = YES # If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen # will put a list of the files that are included by a file in the documentation # of that file. SHOW_INCLUDE_FILES = YES # If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen # will list include files with double quotes in the documentation # rather than with sharp brackets. FORCE_LOCAL_INCLUDES = NO # If the INLINE_INFO tag is set to YES (the default) then a tag [inline] # is inserted in the documentation for inline members. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen # will sort the (detailed) documentation of file and class members # alphabetically by member name. If set to NO the members will appear in # declaration order. SORT_MEMBER_DOCS = YES # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the # brief documentation of file, namespace and class members alphabetically # by member name. If set to NO (the default) the members will appear in # declaration order. SORT_BRIEF_DOCS = NO # If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen # will sort the (brief and detailed) documentation of class members so that # constructors and destructors are listed first. If set to NO (the default) # the constructors will appear in the respective orders defined by # SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. # This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO # and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. SORT_MEMBERS_CTORS_1ST = NO # If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the # hierarchy of group names into alphabetical order. If set to NO (the default) # the group names will appear in their defined order. SORT_GROUP_NAMES = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be # sorted by fully-qualified names, including namespaces. If set to # NO (the default), the class list will be sorted only by class name, # not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the # alphabetical list. SORT_BY_SCOPE_NAME = NO # The GENERATE_TODOLIST tag can be used to enable (YES) or # disable (NO) the todo list. This list is created by putting \todo # commands in the documentation. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable (YES) or # disable (NO) the test list. This list is created by putting \test # commands in the documentation. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable (YES) or # disable (NO) the bug list. This list is created by putting \bug # commands in the documentation. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or # disable (NO) the deprecated list. This list is created by putting # \deprecated commands in the documentation. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional # documentation sections, marked by \if sectionname ... \endif. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines # the initial value of a variable or define consists of for it to appear in # the documentation. If the initializer consists of more lines than specified # here it will be hidden. Use a value of 0 to hide initializers completely. # The appearance of the initializer of individual variables and defines in the # documentation can be controlled using \showinitializer or \hideinitializer # command in the documentation regardless of this setting. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated # at the bottom of the documentation of classes and structs. If set to YES the # list will mention the files that were used to generate the documentation. SHOW_USED_FILES = YES # If the sources in your project are distributed over multiple directories # then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy # in the documentation. The default is NO. SHOW_DIRECTORIES = NO # Set the SHOW_FILES tag to NO to disable the generation of the Files page. # This will remove the Files entry from the Quick Index and from the # Folder Tree View (if specified). The default is YES. SHOW_FILES = YES # Set the SHOW_NAMESPACES tag to NO to disable the generation of the # Namespaces page. This will remove the Namespaces entry from the Quick Index # and from the Folder Tree View (if specified). The default is YES. SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via # popen()) the command , where is the value of # the FILE_VERSION_FILTER tag, and is the name of an input file # provided by doxygen. Whatever the program writes to standard output # is used as the file version. See the manual for examples. FILE_VERSION_FILTER = # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed # by doxygen. The layout file controls the global structure of the generated # output files in an output format independent way. The create the layout file # that represents doxygen's defaults, run doxygen with the -l option. # You can optionally specify a file name after the option, if omitted # DoxygenLayout.xml will be used as the name of the layout file. LAYOUT_FILE = #--------------------------------------------------------------------------- # configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated # by doxygen. Possible values are YES and NO. If left blank NO is used. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated by doxygen. Possible values are YES and NO. If left blank # NO is used. WARNINGS = YES # If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings # for undocumented members. If EXTRACT_ALL is set to YES then this flag will # automatically be disabled. WARN_IF_UNDOCUMENTED = YES # If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some # parameters in a documented function, or documenting parameters that # don't exist or using markup commands wrongly. WARN_IF_DOC_ERROR = YES # This WARN_NO_PARAMDOC option can be abled to get warnings for # functions that are documented, but have no documentation for their parameters # or return value. If set to NO (the default) doxygen will only warn about # wrong or incomplete parameter documentation, but not about the absence of # documentation. WARN_NO_PARAMDOC = NO # The WARN_FORMAT tag determines the format of the warning messages that # doxygen can produce. The string should contain the $file, $line, and $text # tags, which will be replaced by the file and line number from which the # warning originated and the warning text. Optionally the format may contain # $version, which will be replaced by the version of the file (if it could # be obtained via FILE_VERSION_FILTER) WARN_FORMAT = "$file:$line: $text" # The WARN_LOGFILE tag can be used to specify a file to which warning # and error messages should be written. If left blank the output is written # to stderr. WARN_LOGFILE = #--------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag can be used to specify the files and/or directories that contain # documented source files. You may enter file names like "myfile.cpp" or # directories like "/usr/src/myproject". Separate the files or directories # with spaces. INPUT = /home/rger/proj/liblognorm/src # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is # also the default input encoding. Doxygen uses libiconv (or the iconv built # into libc) for the transcoding. See http://www.gnu.org/software/libiconv for # the list of possible encodings. INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank the following patterns are tested: # *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx # *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 FILE_PATTERNS = *.c \ *.cc \ *.cxx \ *.cpp \ *.c++ \ *.d \ *.java \ *.ii \ *.ixx \ *.ipp \ *.i++ \ *.inl \ *.h \ *.hh \ *.hxx \ *.hpp \ *.h++ \ *.idl \ *.odl \ *.cs \ *.php \ *.php3 \ *.inc \ *.m \ *.mm \ *.dox \ *.py \ *.f90 \ *.f \ *.vhd \ *.vhdl # The RECURSIVE tag can be used to turn specify whether or not subdirectories # should be searched for input files as well. Possible values are YES and NO. # If left blank NO is used. RECURSIVE = NO # The EXCLUDE tag can be used to specify files and/or directories that should # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. EXCLUDE = # The EXCLUDE_SYMLINKS tag can be used select whether or not files or # directories that are symbolic links (a Unix filesystem feature) are excluded # from the input. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. Note that the wildcards are matched # against the file with absolute path, so to exclude all test directories # for example use the pattern */test/* EXCLUDE_PATTERNS = # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, # AClass::ANamespace, ANamespace::*Test EXCLUDE_SYMBOLS = # The EXAMPLE_PATH tag can be used to specify one or more files or # directories that contain example code fragments that are included (see # the \include command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank all files are included. EXAMPLE_PATTERNS = * # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude # commands irrespective of the value of the RECURSIVE tag. # Possible values are YES and NO. If left blank NO is used. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or # directories that contain image that are included in the documentation (see # the \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command , where # is the value of the INPUT_FILTER tag, and is the name of an # input file. Doxygen will then use the output that the filter program writes # to standard output. If FILTER_PATTERNS is specified, this tag will be # ignored. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: # pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further # info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER # is applied to all files. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER) will be used to filter the input files when producing source # files to browse (i.e. when SOURCE_BROWSER is set to YES). FILTER_SOURCE_FILES = NO #--------------------------------------------------------------------------- # configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will # be generated. Documented entities will be cross-referenced with these sources. # Note: To get rid of all source code in the generated output, make sure also # VERBATIM_HEADERS is set to NO. SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body # of functions and classes directly in the documentation. INLINE_SOURCES = NO # Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct # doxygen to hide any special comment blocks from generated source code # fragments. Normal C and C++ comments will always remain visible. STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES # then for each documented function all documented # functions referencing it will be listed. REFERENCED_BY_RELATION = NO # If the REFERENCES_RELATION tag is set to YES # then for each documented function all documented entities # called/used by that function will be listed. REFERENCES_RELATION = NO # If the REFERENCES_LINK_SOURCE tag is set to YES (the default) # and SOURCE_BROWSER tag is set to YES, then the hyperlinks from # functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will # link to the source code. Otherwise they will link to the documentation. REFERENCES_LINK_SOURCE = YES # If the USE_HTAGS tag is set to YES then the references to source code # will point to the HTML generated by the htags(1) tool instead of doxygen # built-in source browser. The htags tool is part of GNU's global source # tagging system (see http://www.gnu.org/software/global/global.html). You # will need version 4.8.6 or higher. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen # will generate a verbatim copy of the header file for each class for # which an include is specified. Set to NO to disable this. VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index # of all compounds will be generated. Enable this if the project # contains a lot of classes, structs, unions or interfaces. ALPHABETICAL_INDEX = YES # If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then # the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns # in which this list will be split (can be a number in the range [1..20]) COLS_IN_ALPHA_INDEX = 5 # In case all classes in a project start with a common prefix, all # classes will be put under the same header in the alphabetical index. # The IGNORE_PREFIX tag can be used to specify one or more prefixes that # should be ignored while generating the index headers. IGNORE_PREFIX = #--------------------------------------------------------------------------- # configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES (the default) Doxygen will # generate HTML output. GENERATE_HTML = YES # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `html' will be used as the default path. HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for # each generated HTML page (for example: .htm,.php,.asp). If it is left blank # doxygen will generate files with .html extension. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a personal HTML header for # each generated HTML page. If it is left blank doxygen will generate a # standard header. HTML_HEADER = # The HTML_FOOTER tag can be used to specify a personal HTML footer for # each generated HTML page. If it is left blank doxygen will generate a # standard footer. HTML_FOOTER = # If the HTML_TIMESTAMP tag is set to YES then the generated HTML # documentation will contain the timesstamp. HTML_TIMESTAMP = YES # The HTML_STYLESHEET tag can be used to specify a user-defined cascading # style sheet that is used by each HTML page. It can be used to # fine-tune the look of the HTML output. If the tag is left blank doxygen # will generate a default style sheet. Note that doxygen will try to copy # the style sheet file to the HTML output directory, so don't put your own # stylesheet in the HTML output directory as well, or it will be erased! HTML_STYLESHEET = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. # Doxygen will adjust the colors in the stylesheet and background images # according to this color. Hue is specified as an angle on a colorwheel, # see http://en.wikipedia.org/wiki/Hue for more information. # For instance the value 0 represents red, 60 is yellow, 120 is green, # 180 is cyan, 240 is blue, 300 purple, and 360 is red again. # The allowed range is 0 to 359. HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of # the colors in the HTML output. For a value of 0 the output will use # grayscales only. A value of 255 will produce the most vivid colors. HTML_COLORSTYLE_SAT = 100 # The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to # the luminance component of the colors in the HTML output. Values below # 100 gradually make the output lighter, whereas values above 100 make # the output darker. The value divided by 100 is the actual gamma applied, # so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, # and 100 does not change the gamma. HTML_COLORSTYLE_GAMMA = 80 # If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML # page will contain the date and time when the page was generated. Setting # this to NO can help when comparing the output of multiple runs. HTML_TIMESTAMP = YES # If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, # files or namespaces will be aligned in HTML using tables. If set to # NO a bullet list will be used. HTML_ALIGN_MEMBERS = YES # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. For this to work a browser that supports # JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox # Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). HTML_DYNAMIC_SECTIONS = NO # If the GENERATE_DOCSET tag is set to YES, additional index files # will be generated that can be used as input for Apple's Xcode 3 # integrated development environment, introduced with OSX 10.5 (Leopard). # To create a documentation set, doxygen will generate a Makefile in the # HTML output directory. Running make will produce the docset in that # directory and running "make install" will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find # it at startup. # See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html # for more information. GENERATE_DOCSET = NO # When GENERATE_DOCSET tag is set to YES, this tag determines the name of the # feed. A documentation feed provides an umbrella under which multiple # documentation sets from a single provider (such as a company or product suite) # can be grouped. DOCSET_FEEDNAME = "Doxygen generated docs" # When GENERATE_DOCSET tag is set to YES, this tag specifies a string that # should uniquely identify the documentation set bundle. This should be a # reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen # will append .docset to the name. DOCSET_BUNDLE_ID = org.doxygen.Project # When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style # string, e.g. com.mycompany.MyDocSet.documentation. DOCSET_PUBLISHER_ID = org.doxygen.Publisher # The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES, additional index files # will be generated that can be used as input for tools like the # Microsoft HTML help workshop to generate a compiled HTML help file (.chm) # of the generated HTML documentation. GENERATE_HTMLHELP = NO # If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can # be used to specify the file name of the resulting .chm file. You # can add a path in front of the file if the result should not be # written to the html output directory. CHM_FILE = # If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can # be used to specify the location (absolute path including file name) of # the HTML help compiler (hhc.exe). If non-empty doxygen will try to run # the HTML help compiler on the generated index.hhp. HHC_LOCATION = # If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag # controls if a separate .chi index file is generated (YES) or that # it should be included in the master .chm file (NO). GENERATE_CHI = NO # If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING # is used to encode HtmlHelp index (hhk), content (hhc) and project file # content. CHM_INDEX_ENCODING = # If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag # controls whether a binary table of contents is generated (YES) or a # normal table of contents (NO) in the .chm file. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members # to the contents of the HTML help documentation and to the tree view. TOC_EXPAND = NO # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated # that can be used as input for Qt's qhelpgenerator to generate a # Qt Compressed Help (.qch) of the generated HTML documentation. GENERATE_QHP = NO # If the QHG_LOCATION tag is specified, the QCH_FILE tag can # be used to specify the file name of the resulting .qch file. # The path specified is relative to the HTML output folder. QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating # Qt Help Project output. For more information please see # http://doc.trolltech.com/qthelpproject.html#namespace QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating # Qt Help Project output. For more information please see # http://doc.trolltech.com/qthelpproject.html#virtual-folders QHP_VIRTUAL_FOLDER = doc # If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to # add. For more information please see # http://doc.trolltech.com/qthelpproject.html#custom-filters QHP_CUST_FILTER_NAME = # The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see # # Qt Help Project / Custom Filters. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's # filter section matches. # # Qt Help Project / Filter Attributes. QHP_SECT_FILTER_ATTRS = # If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can # be used to specify the location of Qt's qhelpgenerator. # If non-empty doxygen will try to run qhelpgenerator on the generated # .qhp file. QHG_LOCATION = # If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files # will be generated, which together with the HTML files, form an Eclipse help # plugin. To install this plugin and make it available under the help contents # menu in Eclipse, the contents of the directory containing the HTML and XML # files needs to be copied into the plugins directory of eclipse. The name of # the directory within the plugins directory should be the same as # the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before # the help appears. GENERATE_ECLIPSEHELP = NO # A unique identifier for the eclipse help plugin. When installing the plugin # the directory name containing the HTML and XML files should also have # this name. ECLIPSE_DOC_ID = org.doxygen.Project # The DISABLE_INDEX tag can be used to turn on/off the condensed index at # top of each HTML page. The value NO (the default) enables the index and # the value YES disables it. DISABLE_INDEX = NO # This tag can be used to set the number of enum values (range [1..20]) # that doxygen will group on one line in the generated HTML documentation. ENUM_VALUES_PER_LINE = 4 # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. # If the tag value is set to YES, a side panel will be generated # containing a tree-like index structure (just like the one that # is generated for HTML Help). For this to work a browser that supports # JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). # Windows users are probably better off using the HTML help feature. GENERATE_TREEVIEW = NO # By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, # and Class Hierarchy pages using a tree view instead of an ordered list. USE_INLINE_TREES = NO # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be # used to set the initial width (in pixels) of the frame in which the tree # is shown. TREEVIEW_WIDTH = 250 # When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open # links to external symbols imported via tag files in a separate window. EXT_LINKS_IN_WINDOW = NO # Use this tag to change the font size of Latex formulas included # as images in the HTML documentation. The default is 10. Note that # when you change the font size after a successful doxygen run you need # to manually remove any form_*.png images from the HTML output directory # to force them to be regenerated. FORMULA_FONTSIZE = 10 # Use the FORMULA_TRANPARENT tag to determine whether or not the images # generated for formulas are transparent PNGs. Transparent PNGs are # not supported properly for IE 6.0, but are supported on all modern browsers. # Note that when changing this option you need to delete any form_*.png files # in the HTML output before the changes have effect. FORMULA_TRANSPARENT = YES # When the SEARCHENGINE tag is enabled doxygen will generate a search box # for the HTML output. The underlying search engine uses javascript # and DHTML and should work on any modern browser. Note that when using # HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets # (GENERATE_DOCSET) there is already a search function so this one should # typically be disabled. For large projects the javascript based search engine # can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. SEARCHENGINE = NO # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a PHP enabled web server instead of at the web client # using Javascript. Doxygen will generate the search PHP script and index # file to put on the web server. The advantage of the server # based approach is that it scales better to large projects and allows # full text search. The disadvances is that it is more difficult to setup # and does not have live searching capabilities. SERVER_BASED_SEARCH = NO #--------------------------------------------------------------------------- # configuration options related to the LaTeX output #--------------------------------------------------------------------------- # If the GENERATE_LATEX tag is set to YES (the default) Doxygen will # generate Latex output. GENERATE_LATEX = NO # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `latex' will be used as the default path. LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. If left blank `latex' will be used as the default command name. # Note that when enabling USE_PDFLATEX this option is only used for # generating bitmaps for formulas in the HTML output, but not in the # Makefile that is written to the output directory. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to # generate index for LaTeX. If left blank `makeindex' will be used as the # default command name. MAKEINDEX_CMD_NAME = makeindex # If the COMPACT_LATEX tag is set to YES Doxygen generates more compact # LaTeX documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_LATEX = NO # The PAPER_TYPE tag can be used to set the paper type that is used # by the printer. Possible values are: a4, a4wide, letter, legal and # executive. If left blank a4wide will be used. PAPER_TYPE = a4wide # The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX # packages that should be included in the LaTeX output. EXTRA_PACKAGES = # The LATEX_HEADER tag can be used to specify a personal LaTeX header for # the generated latex document. The header should contain everything until # the first chapter. If it is left blank doxygen will generate a # standard header. Notice: only use this tag if you know what you are doing! LATEX_HEADER = # If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated # is prepared for conversion to pdf (using ps2pdf). The pdf file will # contain links (just like the HTML output) instead of page references # This makes the output suitable for online browsing using a pdf viewer. PDF_HYPERLINKS = YES # If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of # plain latex in the generated Makefile. Set this option to YES to get a # higher quality PDF documentation. USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. # command to the generated LaTeX files. This will instruct LaTeX to keep # running if errors occur, instead of asking the user for help. # This option is also used when generating formulas in HTML. LATEX_BATCHMODE = NO # If LATEX_HIDE_INDICES is set to YES then doxygen will not # include the index chapters (such as File Index, Compound Index, etc.) # in the output. LATEX_HIDE_INDICES = NO # If LATEX_SOURCE_CODE is set to YES then doxygen will include # source code with syntax highlighting in the LaTeX output. # Note that which sources are shown also depends on other settings # such as SOURCE_BROWSER. LATEX_SOURCE_CODE = NO #--------------------------------------------------------------------------- # configuration options related to the RTF output #--------------------------------------------------------------------------- # If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output # The RTF output is optimized for Word 97 and may not look very pretty with # other RTF readers or editors. GENERATE_RTF = NO # The RTF_OUTPUT tag is used to specify where the RTF docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `rtf' will be used as the default path. RTF_OUTPUT = rtf # If the COMPACT_RTF tag is set to YES Doxygen generates more compact # RTF documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_RTF = NO # If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated # will contain hyperlink fields. The RTF file will # contain links (just like the HTML output) instead of page references. # This makes the output suitable for online browsing using WORD or other # programs which support those fields. # Note: wordpad (write) and others do not support links. RTF_HYPERLINKS = NO # Load stylesheet definitions from file. Syntax is similar to doxygen's # config file, i.e. a series of assignments. You only have to provide # replacements, missing definitions are set to their default value. RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an rtf document. # Syntax is similar to doxygen's config file. RTF_EXTENSIONS_FILE = #--------------------------------------------------------------------------- # configuration options related to the man page output #--------------------------------------------------------------------------- # If the GENERATE_MAN tag is set to YES (the default) Doxygen will # generate man pages GENERATE_MAN = NO # The MAN_OUTPUT tag is used to specify where the man pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `man' will be used as the default path. MAN_OUTPUT = man # The MAN_EXTENSION tag determines the extension that is added to # the generated man pages (default is the subroutine's section .3) MAN_EXTENSION = .3 # If the MAN_LINKS tag is set to YES and Doxygen generates man output, # then it will generate one additional man file for each entity # documented in the real man page(s). These additional files # only source the real man page, but without them the man command # would be unable to find the correct page. The default is NO. MAN_LINKS = NO #--------------------------------------------------------------------------- # configuration options related to the XML output #--------------------------------------------------------------------------- # If the GENERATE_XML tag is set to YES Doxygen will # generate an XML file that captures the structure of # the code including all documentation. GENERATE_XML = NO # The XML_OUTPUT tag is used to specify where the XML pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `xml' will be used as the default path. XML_OUTPUT = xml # The XML_SCHEMA tag can be used to specify an XML schema, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_SCHEMA = # The XML_DTD tag can be used to specify an XML DTD, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_DTD = # If the XML_PROGRAMLISTING tag is set to YES Doxygen will # dump the program listings (including syntax highlighting # and cross-referencing information) to the XML output. Note that # enabling this will significantly increase the size of the XML output. XML_PROGRAMLISTING = YES #--------------------------------------------------------------------------- # configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will # generate an AutoGen Definitions (see autogen.sf.net) file # that captures the structure of the code including all # documentation. Note that this feature is still experimental # and incomplete at the moment. GENERATE_AUTOGEN_DEF = NO #--------------------------------------------------------------------------- # configuration options related to the Perl module output #--------------------------------------------------------------------------- # If the GENERATE_PERLMOD tag is set to YES Doxygen will # generate a Perl module file that captures the structure of # the code including all documentation. Note that this # feature is still experimental and incomplete at the # moment. GENERATE_PERLMOD = NO # If the PERLMOD_LATEX tag is set to YES Doxygen will generate # the necessary Makefile rules, Perl scripts and LaTeX code to be able # to generate PDF and DVI output from the Perl module output. PERLMOD_LATEX = NO # If the PERLMOD_PRETTY tag is set to YES the Perl module output will be # nicely formatted so it can be parsed by a human reader. This is useful # if you want to understand what is going on. On the other hand, if this # tag is set to NO the size of the Perl module output will be much smaller # and Perl will parse it just the same. PERLMOD_PRETTY = YES # The names of the make variables in the generated doxyrules.make file # are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. # This is useful so different doxyrules.make files included by the same # Makefile don't overwrite each other's variables. PERLMOD_MAKEVAR_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- # If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will # evaluate all C-preprocessor directives found in the sources and include # files. ENABLE_PREPROCESSING = YES # If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro # names in the source code. If set to NO (the default) only conditional # compilation will be performed. Macro expansion can be done in a controlled # way by setting EXPAND_ONLY_PREDEF to YES. MACRO_EXPANSION = NO # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES # then the macro expansion is limited to the macros specified with the # PREDEFINED and EXPAND_AS_DEFINED tags. EXPAND_ONLY_PREDEF = NO # If the SEARCH_INCLUDES tag is set to YES (the default) the includes files # in the INCLUDE_PATH (see below) will be search if a #include is found. SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by # the preprocessor. INCLUDE_PATH = # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard # patterns (like *.h and *.hpp) to filter out the header-files in the # directories. If left blank, the patterns specified with FILE_PATTERNS will # be used. INCLUDE_FILE_PATTERNS = # The PREDEFINED tag can be used to specify one or more macro names that # are defined before the preprocessor is started (similar to the -D option of # gcc). The argument of the tag is a list of macros of the form: name # or name=definition (no spaces). If the definition and the = are # omitted =1 is assumed. To prevent a macro definition from being # undefined via #undef or recursively expanded use the := operator # instead of the = operator. PREDEFINED = # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then # this tag can be used to specify a list of macro names that should be expanded. # The macro definition that is found in the sources will be used. # Use the PREDEFINED tag if you want to use a different macro definition. EXPAND_AS_DEFINED = # If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then # doxygen's preprocessor will remove all function-like macros that are alone # on a line, have an all uppercase name, and do not end with a semicolon. Such # function macros are typically used for boiler-plate code, and will confuse # the parser if not removed. SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration::additions related to external references #--------------------------------------------------------------------------- # The TAGFILES option can be used to specify one or more tagfiles. # Optionally an initial location of the external documentation # can be added for each tagfile. The format of a tag file without # this location is as follows: # TAGFILES = file1 file2 ... # Adding location for the tag files is done as follows: # TAGFILES = file1=loc1 "file2 = loc2" ... # where "loc1" and "loc2" can be relative or absolute paths or # URLs. If a location is present for each tag, the installdox tool # does not have to be run to correct the links. # Note that each tag file must have a unique name # (where the name does NOT include the path) # If a tag file is not located in the directory in which doxygen # is run, you must also specify the path to the tagfile here. TAGFILES = # When a file name is specified after GENERATE_TAGFILE, doxygen will create # a tag file that is based on the input files it reads. GENERATE_TAGFILE = # If the ALLEXTERNALS tag is set to YES all external classes will be listed # in the class index. If set to NO only the inherited external classes # will be listed. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed # in the modules index. If set to NO, only the current project's groups will # be listed. EXTERNAL_GROUPS = YES # The PERL_PATH should be the absolute path and name of the perl script # interpreter (i.e. the result of `which perl'). PERL_PATH = /usr/bin/perl #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- # If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will # generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base # or super classes. Setting the tag to NO turns the diagrams off. Note that # this option is superseded by the HAVE_DOT option below. This is only a # fallback. It is recommended to install and use dot, since it yields more # powerful graphs. CLASS_DIAGRAMS = YES # You can define message sequence charts within doxygen comments using the \msc # command. Doxygen will then run the mscgen tool (see # http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the # documentation. The MSCGEN_PATH tag allows you to specify the directory where # the mscgen tool resides. If left empty the tool is assumed to be found in the # default search path. MSCGEN_PATH = # If set to YES, the inheritance and collaboration graphs will hide # inheritance and usage relations if the target is undocumented # or is not a class. HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz, a graph visualization # toolkit from AT&T and Lucent Bell Labs. The other options in this section # have no effect if this option is set to NO (the default) HAVE_DOT = NO # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is # allowed to run in parallel. When set to 0 (the default) doxygen will # base this on the number of processors available in the system. You can set it # explicitly to a value larger than 0 to get control over the balance # between CPU load and processing speed. DOT_NUM_THREADS = 0 # By default doxygen will write a font called FreeSans.ttf to the output # directory and reference it in all dot files that doxygen generates. This # font does not include all possible unicode characters however, so when you need # these (or just want a differently looking font) you can specify the font name # using DOT_FONTNAME. You need need to make sure dot is able to find the font, # which can be done by putting it in a standard location or by setting the # DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory # containing the font. DOT_FONTNAME = FreeSans.ttf # The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. # The default size is 10pt. DOT_FONTSIZE = 10 # By default doxygen will tell dot to use the output directory to look for the # FreeSans.ttf font (which doxygen will put there itself). If you specify a # different font using DOT_FONTNAME you can set the path where dot # can find it using this tag. DOT_FONTPATH = # If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect inheritance relations. Setting this tag to YES will force the # the CLASS_DIAGRAMS tag to NO. CLASS_GRAPH = YES # If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect implementation dependencies (inheritance, containment, and # class references variables) of the class with other documented classes. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen # will generate a graph for groups, showing the direct groups dependencies GROUP_GRAPHS = YES # If the UML_LOOK tag is set to YES doxygen will generate inheritance and # collaboration diagrams in a style similar to the OMG's Unified Modeling # Language. UML_LOOK = NO # If set to YES, the inheritance and collaboration graphs will show the # relations between templates and their instances. TEMPLATE_RELATIONS = NO # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT # tags are set to YES then doxygen will generate a graph for each documented # file showing the direct and indirect include dependencies of the file with # other documented files. INCLUDE_GRAPH = YES # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and # HAVE_DOT tags are set to YES then doxygen will generate a graph for each # documented header file showing the documented files that directly or # indirectly include this file. INCLUDED_BY_GRAPH = YES # If the CALL_GRAPH and HAVE_DOT options are set to YES then # doxygen will generate a call dependency graph for every global function # or class method. Note that enabling this option will significantly increase # the time of a run. So in most cases it will be better to enable call graphs # for selected functions only using the \callgraph command. CALL_GRAPH = NO # If the CALLER_GRAPH and HAVE_DOT tags are set to YES then # doxygen will generate a caller dependency graph for every global function # or class method. Note that enabling this option will significantly increase # the time of a run. So in most cases it will be better to enable caller # graphs for selected functions only using the \callergraph command. CALLER_GRAPH = NO # If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen # will graphical hierarchy of all classes instead of a textual one. GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES # then doxygen will show the dependencies a directory has on other directories # in a graphical way. The dependency relations are determined by the #include # relations between the files in the directories. DIRECTORY_GRAPH = YES # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. Possible values are png, jpg, or gif # If left blank png will be used. DOT_IMAGE_FORMAT = png # The tag DOT_PATH can be used to specify the path where the dot tool can be # found. If left blank, it is assumed the dot tool can be found in the path. DOT_PATH = # The DOTFILE_DIRS tag can be used to specify one or more directories that # contain dot files that are included in the documentation (see the # \dotfile command). DOTFILE_DIRS = # The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of # nodes that will be shown in the graph. If the number of nodes in a graph # becomes larger than this value, doxygen will truncate the graph, which is # visualized by representing a node as a red box. Note that doxygen if the # number of direct children of the root node in a graph is already larger than # DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note # that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. DOT_GRAPH_MAX_NODES = 50 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the # graphs generated by dot. A depth value of 3 means that only nodes reachable # from the root by following a path via at most 3 edges will be shown. Nodes # that lay further from the root node will be omitted. Note that setting this # option to 1 or 2 may greatly reduce the computation time needed for large # code bases. Also note that the size of a graph can be further restricted by # DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. MAX_DOT_GRAPH_DEPTH = 0 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent # background. This is disabled by default, because dot on Windows does not # seem to support this out of the box. Warning: Depending on the platform used, # enabling this option may lead to badly anti-aliased labels on the edges of # a graph (i.e. they become hard to read). DOT_TRANSPARENT = NO # Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) # support this, this feature is disabled by default. DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will # generate a legend page explaining the meaning of the various boxes and # arrows in the dot generated graphs. GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES (the default) Doxygen will # remove the intermediate dot files that are used to generate # the various graphs. DOT_CLEANUP = YES liblognorm-2.0.8/Makefile.am000066400000000000000000000003641511425433100157230ustar00rootroot00000000000000SUBDIRS = compat src tools if ENABLE_DOCS SUBDIRS += doc endif EXTRA_DIST = rulebases \ COPYING.ASL20 pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = lognorm.pc ACLOCAL_AMFLAGS = -I m4 if ENABLE_TESTBENCH SUBDIRS += tests endif liblognorm-2.0.8/NEWS000066400000000000000000000000771511425433100143670ustar00rootroot00000000000000This file has been superseeded by ChangeLog. Please see there. liblognorm-2.0.8/README000066400000000000000000000037571511425433100145600ustar00rootroot00000000000000Liblognorm is a fast-samples based normalization library. More information on liblognorm can be found at http://www.liblognorm.com Liblognorm evolves since several years and was initially meant to be used primarily with the Mitre CEE effort. Consequently, the initial version of liblognorm (0.x) uses the libee CEE support library in its API. As time evolved, the initial CEE schema underwent considerable change. Even worse, Mitre lost funding for CEE. While the CEE ideas survived as part of Red Hat-driven "Project Lumberjack", the data structures became greatly simplified and JSON based. That effectively made libee obsolete (and also in parts libestr, which was specifically written to support CEE's initial requirement of embedded NUL chars in strings). In 2013, Pavel Levshin converted liblognorm to native JSON, which helped improve performance and simplicity for many client applications. Unfortunately, this change broke interface compatibility (and there was no way to avoid that, obviously...). In 2015, most parts of liblognorm were redesigned and rewritten as part of Rainer Gerhards' master thesis. For full technical details of how liblognorm operates, and why it is so fast, please have a look at https://www.researchgate.net/publication/310545144_Efficient_Normalization_of_IT_Log_Messages_under_Realtime_Conditions The current library is the result of that effort. Application developers are encouraged to switch to this version, as it provides the benefit of a simpler API. This version is now being tracked by the git default branch. However, if you need to stick to the old API, there is a git branch liblognorm0, which contains the previous version of the library. This branch is also maintained for important bug fixes, so it is safe to use. We recommend that packagers create packages both for liblognorm0 and liblognorm1. Note that liblognorm's development packages cannot coexist on the same system as the PKGCONFIG system would get into trouble. Adiscon's own packages follow this schema. liblognorm-2.0.8/compat/000077500000000000000000000000001511425433100151475ustar00rootroot00000000000000liblognorm-2.0.8/compat/Makefile.am000066400000000000000000000003341511425433100172030ustar00rootroot00000000000000noinst_LTLIBRARIES = compat.la compat_la_SOURCES = strndup.c asprintf.c compat_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) compat_la_LDFLAGS = -module -avoid-version compat_la_LIBADD = $(IMUDP_LIBS) liblognorm-2.0.8/compat/asprintf.c000066400000000000000000000025071511425433100171450ustar00rootroot00000000000000/* compatibility file for systems without asprintf. * * Copyright 2015 Rainer Gerhards and Adiscon * * This file is part of rsyslog. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "config.h" #ifndef HAVE_ASPRINTF #include #include #include int asprintf(char **strp, const char *fmt, ...) { va_list ap; int len; va_start(ap, fmt); len = vsnprintf(NULL, 0, fmt, ap); va_end(ap); *strp = malloc(len+1); if (!*strp) { return -1; } va_start(ap, fmt); vsnprintf(*strp, len+1, fmt, ap); va_end(ap); (*strp)[len] = 0; return len; } #else /* XLC needs at least one method in source file even static to compile */ #ifdef __xlc__ static void dummy() {} #endif #endif /* #ifndef HAVE_ASPRINTF */ liblognorm-2.0.8/compat/strndup.c000066400000000000000000000024321511425433100170130ustar00rootroot00000000000000/* compatibility file for systems without strndup. * * Copyright 2015 Rainer Gerhards and Adiscon * * This file is part of liblognorm. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * -or- * see COPYING.ASL20 in the source distribution * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "config.h" #ifndef HAVE_STRNDUP #include #include char * strndup(const char *s, size_t n) { const size_t len = strlen(s); if(len <= n) return strdup(s); char *const new_s = malloc(len+1); if(new_s == NULL) return NULL; memcpy(new_s, s, len); new_s[len] = '\0'; return new_s; } #else /* #ifndef HAVE_STRNDUP */ /* Solaris must have at least one symbol inside a file, so we provide * it here ;-) */ void dummy_dummy_required_for_solaris_do_not_use(void) { } #endif /* #ifndef HAVE_STRNDUP */ liblognorm-2.0.8/compatibility-v2000066400000000000000000000032701511425433100170070ustar00rootroot00000000000000- the lognormalizer tool now defaults to json output format - API changes * ln_loadSample() does no longer exist loading of samples at runtime is no longer possible. This is due to the fact that the parse DAG is compiled into the most efficient form and thereafter read-only. If urgently needed, we could change the code so that the unoptimized form is kept (memory intense) and a re-compile happens after each addition (time intense). - the plain old iptables parser is no longer supported. Sorry for that, but keeping it would have caused violation of layers. It can be replaced by the new one - the rulebase format for v2 has changed to support the enhancements It is mostly compatible with v1, which means it understands mosts of it's constructs. However, some are NOT understood: - the "tokenized" parser is no longer supported. Use the more capable "repeat" parser instead. - the "recursive" and "descent" parsers are no longer supported. Use user-defined data types instead. - the suffixed parser and friends are no longer supported. A replacement is currently being developed. - the regexp parser is no longer supported and needs to be replaced by other liblognorm features for details on this decision see: https://github.com/rsyslog/liblognorm/issues/143 To signify that a rule base file contains v2 format, it must contain the line version=2 as the very first line of the file, in exactly this format (no comments, no whitespace, nothing else in between the words or after them). If that line is missing or not given **exactly** as above, the old v1 engine is used, with all of its restrictions. If it is present, the v2 engine is used. liblognorm-2.0.8/configure.ac000066400000000000000000000140741511425433100161600ustar00rootroot00000000000000 -*- Autoconf -*- # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) AC_INIT([liblognorm], [2.0.8], [rgerhards@adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AC_CONFIG_SRCDIR([src/lognorm.c]) AC_CONFIG_HEADER([config.h]) AC_USE_SYSTEM_EXTENSIONS # Checks for programs. AC_PROG_CC AM_PROG_CC_C_O AC_PROG_CC_C99 AC_PROG_LIBTOOL m4_ifdef([AX_IS_RELEASE], [ AX_IS_RELEASE([git-directory]) m4_ifdef([AX_COMPILER_FLAGS], [ AX_COMPILER_FLAGS() ], [ if test "$GCC" = "yes" then CFLAGS="$CFLAGS -W -Wall -Wformat-security -Wshadow -Wcast-align -Wpointer-arith -Wmissing-format-attribute -g" fi AC_MSG_WARN([missing AX_COMPILER_FLAGS macro, not using it]) ]) ], [ if test "$GCC" = "yes" then CFLAGS="$CFLAGS -W -Wall -Wformat-security -Wshadow -Wcast-align -Wpointer-arith -Wmissing-format-attribute -g" fi AC_MSG_WARN([missing AX_IS_RELEASE macro, not using AX_COMPILER_FLAGS macro because of this]) ]) # Checks for libraries. save_LIBS=$LIBS LIBS= AC_SEARCH_LIBS(clock_getm4_defn([AC_AUTOCONF_VERSION]), [2.68]time, rt) LIBS=$save_LIBS # Checks for header files. AC_HEADER_STDC #AC_CHECK_HEADERS([]) # Checks for typedefs, structures, and compiler characteristics. AC_C_CONST AC_TYPE_SIZE_T #AC_HEADER_TIME #AC_STRUCT_TM # Checks for library functions. AC_FUNC_SELECT_ARGTYPES AC_TYPE_SIGNAL AC_FUNC_STRERROR_R AC_CHECK_FUNCS([strdup strndup strtok_r]) LIBLOGNORM_CFLAGS="-I\$(top_srcdir)/src" LIBLOGNORM_LIBS="\$(top_builddir)/src/liblognorm.la" AC_SUBST(LIBLOGNORM_CFLAGS) AC_SUBST(LIBLOGNORM_LIBS) # modules we require PKG_CHECK_MODULES(LIBESTR, libestr >= 0.0.0) PKG_CHECK_MODULES(JSON_C, libfastjson,, ) # add libestr flags to pkgconfig file for static linking AC_SUBST(pkg_config_libs_private, $LIBESTR_LIBS) # Regular expressions AC_ARG_ENABLE(regexp, [AS_HELP_STRING([--enable-regexp],[Enable regular expressions support @<:@default=no@:>@])], [case "${enableval}" in yes) enable_regexp="yes" ;; no) enable_regexp="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-regexp) ;; esac], [enable_regexp="no"] ) AM_CONDITIONAL(ENABLE_REGEXP, test x$enable_regexp = xyes) if test "$enable_regexp" = "yes"; then PKG_CHECK_MODULES(PCRE, libpcre) AC_DEFINE(FEATURE_REGEXP, 1, [Regular expressions support enabled.]) FEATURE_REGEXP=1 else FEATURE_REGEXP=0 fi AC_SUBST(FEATURE_REGEXP) # debug mode settings AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],[Enable debug mode @<:@default=no@:>@])], [case "${enableval}" in yes) enable_debug="yes" ;; no) enable_debug="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-debug) ;; esac], [enable_debug="no"] ) if test "$enable_debug" = "yes"; then AC_DEFINE(DEBUG, 1, [Defined if debug mode is enabled.]) fi if test "$enable_debug" = "no"; then AC_DEFINE(NDEBUG, 1, [Defined if debug mode is disabled.]) fi # advanced statistics AC_ARG_ENABLE(advanced-stats, [AS_HELP_STRING([--enable-advanced-stats],[Enable advanced statistics @<:@default=no@:>@])], [case "${enableval}" in yes) enable_advstats="yes" ;; no) enable_advstats="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-advstats) ;; esac], [enable_advstats="no"] ) if test "$enable_advstats" = "yes"; then AC_DEFINE(ADVANCED_STATS, 1, [Defined if advanced statistics are enabled.]) fi # docs (html) build settings AC_ARG_ENABLE(docs, [AS_HELP_STRING([--enable-docs],[Enable building HTML docs (requires Sphinx) @<:@default=no@:>@])], [case "${enableval}" in yes) enable_docs="yes" ;; no) enable_docs="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-docs) ;; esac], [enable_docs="no"] ) if test "$enable_docs" = "yes"; then AC_CHECK_PROGS([SPHINXBUILD], [sphinx-build sphinx-build3 sphinx-build2], [no]) if test "$SPHINXBUILD" = "no"; then AC_MSG_ERROR([sphinx-build is required to build documentation, install it or try --disable-docs]) fi fi AM_CONDITIONAL([ENABLE_DOCS], [test "$enable_docs" = "yes"]) AC_ARG_ENABLE(testbench, [AS_HELP_STRING([--enable-testbench],[testbench enabled @<:@default=yes@:>@])], [case "${enableval}" in yes) enable_testbench="yes" ;; no) enable_testbench="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-testbench) ;; esac], [enable_testbench=yes] ) AM_CONDITIONAL(ENABLE_TESTBENCH, test x$enable_testbench = xyes) AC_ARG_ENABLE(valgrind, [AS_HELP_STRING([--enable-valgrind],[valgrind enabled @<:@default=no@:>@])], [case "${enableval}" in yes) enable_valgrind="yes" ;; no) enable_valgrind=="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-valgrind) ;; esac], [enable_valgrind=no] ) AM_CONDITIONAL(ENABLE_VALGRIND, test x$enable_valgrind = xyes) VALGRIND="$enable_valgrind" AC_SUBST(VALGRIND) AC_ARG_ENABLE(tools, [AS_HELP_STRING([--enable-tools],[lognorm toolset enabled @<:@default=yes@:>@])], [case "${enableval}" in yes) enable_tools="yes" ;; no) enable_tools="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-tools) ;; esac], [enable_tools=yes] ) AM_CONDITIONAL(ENABLE_TOOLS, test x$enable_tools = xyes) AC_CONFIG_FILES([Makefile \ lognorm.pc \ compat/Makefile \ doc/Makefile \ src/Makefile \ src/lognorm-features.h \ tools/Makefile \ tests/Makefile \ tests/options.sh]) AC_OUTPUT AC_CONFIG_MACRO_DIR([m4]) echo "*****************************************************" echo "liblognorm will be compiled with the following settings:" echo echo "Regex enabled: $enable_regexp" echo "Advanced Statistics enabled: $enable_advstats" echo "Testbench enabled: $enable_testbench" echo "Valgrind enabled: $enable_valgrind" echo "Debug mode enabled: $enable_debug" echo "Tools enabled: $enable_tools" echo "Docs enabled: $enable_docs" liblognorm-2.0.8/devtools/000077500000000000000000000000001511425433100155235ustar00rootroot00000000000000liblognorm-2.0.8/devtools/default_dev_container000066400000000000000000000000461511425433100217720ustar00rootroot00000000000000rsyslog/rsyslog_dev_base_ubuntu:20.04 liblognorm-2.0.8/devtools/devcontainer.sh000077500000000000000000000042201511425433100205410ustar00rootroot00000000000000#!/bin/bash # This scripts uses an rsyslog development container to execute given # command inside it. # Note: command line parameters are passed as parameters to the container, # with the notable exception that -ti, if given as first parameter, is # passed to "docker run" itself but NOT the container. # # use env var DOCKER_RUN_EXTRA_OPTS to provide extra options to docker run # command. # # # TO MODIFIY BEHAVIOUR, use # LIBLOGNORM_CONTAINER_UID, format uid:gid, # to change the users container is run under # set to "" to use the container default settings # (no local mapping) set -e if [ "$1" == "--rm" ]; then optrm="--rm" shift 1 fi if [ "$1" == "-ti" ]; then ti="-ti" shift 1 fi # check in case -ti was in front... if [ "$1" == "--rm" ]; then optrm="--rm" shift 1 fi if [ "$LIBLOGNORM_HOME" == "" ]; then export LIBLOGNORM_HOME=$(pwd) echo info: LIBLOGNORM_HOME not set, using $LIBLOGNORM_HOME fi if [ -z "$LIBLOGNORM_DEV_CONTAINER" ]; then LIBLOGNORM_DEV_CONTAINER=$(cat $LIBLOGNORM_HOME/devtools/default_dev_container) fi printf '/rsyslog is mapped to %s \n' "$LIBLOGNORM_HOME" printf 'using container %s\n' "$LIBLOGNORM_DEV_CONTAINER" printf 'pulling container...\n' printf 'user ids: %s:%s\n' $(id -u) $(id -g) printf 'container_uid: %s\n' ${LIBLOGNORM_CONTAINER_UID--u $(id -u):$(id -g)} printf 'container cmd: %s\n' $* printf '\nNote: we use the RSYSLOG CONTAINERS, as such project home is /rsyslog!\n\n' docker pull $LIBLOGNORM_DEV_CONTAINER docker run $ti $optrm $DOCKER_RUN_EXTRA_OPTS \ -e LIBLOGNORM_CONFIGURE_OPTIONS_EXTRA \ -e LIBLOGNORM_CONFIGURE_OPTIONS_OVERRIDE \ -e CC \ -e CFLAGS \ -e LDFLAGS \ -e LSAN_OPTIONS \ -e TSAN_OPTIONS \ -e UBSAN_OPTIONS \ -e CI_MAKE_OPT \ -e CI_MAKE_CHECK_OPT \ -e CI_CHECK_CMD \ -e CI_BUILD_URL \ -e CI_CODECOV_TOKEN \ -e CI_VALGRIND_SUPPRESSIONS \ -e CI_SANITIZE_BLACKLIST \ -e ABORT_ALL_ON_TEST_FAIL \ -e USE_AUTO_DEBUG \ -e LIBLOGNORM_STATSURL \ -e VCS_SLUG \ --cap-add SYS_ADMIN \ --cap-add SYS_PTRACE \ ${LIBLOGNORM_CONTAINER_UID--u $(id -u):$(id -g)} \ $DOCKER_RUN_EXTRA_FLAGS \ -v "$LIBLOGNORM_HOME":/rsyslog $LIBLOGNORM_DEV_CONTAINER $* liblognorm-2.0.8/devtools/gather-check-logs.sh000077500000000000000000000036001511425433100213500ustar00rootroot00000000000000#!/bin/bash # gather logs generated by "make [dist]check" # this also limits log size so that buildbot does not abort # Copyright (C) 2020 by Rainer Gerhards, released under ASL 2.0 show_log() { if grep -q ":test-result: FAIL" "$1"; then printf "\nFAIL: ${1%%.trs} \ ########################################################\ ################################\n\n" logfile="${1%%trs}log" if [ -f "$logfile" ]; then lines="$(wc -l < $logfile)" if (( lines > 4000 )); then ls -l $logfile printf 'file is very large (%d lines), showing parts\n' $lines head -n 2000 < "$logfile" printf '\n\n... snip ...\n\n' tail -n 2000 < "$logfile" else cat "$logfile" fi else printf 'log FILE MISSING!\n' fi fi } append_summary() { echo file: $1 # emit file name just in case we have multiple! head -n12 "$1" } # find logs from tests which are potentially aborted. The main indication is # that no matching .trs file exists check_incomplete_logs() { if grep -q "\.dep_wrk\|rstb_\|config.log" <<<"$1"; then return fi # we emit info only for test log files - this means there must # be a matching .sh file by our conventions if [ -f "${1%%log}sh" ]; then trsfile="${1%%log}trs" if [ ! -f "$trsfile" ]; then printf '\n\nNo matching .trs file for %s\n' "$1" ls -l ${1%%.log}* cat "$1" fi fi } export -f show_log export -f append_summary export -f check_incomplete_logs ############################## MAIN ENTRY POINT ############################## printf 'find failing tests\n' rm -f failed-tests.log find . -name "*.trs" -exec bash -c 'show_log "$1" >> failed-tests.log' _ {} \; find . -name "*.log" -exec bash -c 'check_incomplete_logs "$1" >> failed-tests.log' _ {} \; if [ -f failed-tests.log ]; then # show summary stats so that we know how many failed find . -name test-suite.log -exec bash -c 'append_summary "$1" >>failed-tests.log' _ {} \; fi liblognorm-2.0.8/devtools/run-ci.sh000077500000000000000000000036301511425433100172610ustar00rootroot00000000000000#!/bin/bash # script for generic CI runs via container printf 'running CI with\n' printf 'container: %s\n' $RSYSLOG_DEV_CONTAINER printf 'CC:\t%s\n' "$CC" printf 'CFLAGS:\t%s:\n' "$CFLAGS" printf 'RSYSLOG_CONFIGURE_OPTIONS:\t%s\n' "$RSYSLOG_CONFIGURE_OPTIONS" printf 'working directory: %s\n' "$(pwd)" printf 'user ids: %s:%s\n' $(id -u) $(id -g) if [ "$SUDO" != "" ]; then printf 'check sudo' $SUDO echo sudo works! fi if [ "$CI_VALGRIND_SUPPRESSIONS" != "" ]; then export RS_TESTBENCH_VALGRIND_EXTRA_OPTS="--suppressions=$(pwd)/tests/CI/$CI_VALGRIND_SUPPRESSIONS" fi if [ "$CI_SANITIZE_BLACKLIST" != "" ]; then export CFLAGS="$CFLAGS -fsanitize-blacklist=$(pwd)/$CI_SANITIZE_BLACKLIST" printf 'CFLAGS changed to: %s\n', "$CFLAGS" fi set -e printf 'STEP: autoreconf / configure ===============================================\n' autoreconf -fvi ./configure if [ "$CI_CHECK_CMD" != "distcheck" ]; then printf 'STEP: make =================================================================\n' make $CI_MAKE_OPT fi printf 'STEP: make %s ==============================================================\n', \ "$CI_CHECK_CMD" set +e echo CI_CHECK_CMD: $CI_CHECK_CMD make $CI_MAKE_CHECK_OPT ${CI_CHECK_CMD:-check} rc=$? printf 'STEP: find failing tests ====================================================\n' echo calling gather-check-logs devtools/gather-check-logs.sh printf 'STEP: Codecov upload =======================================================\n' if [ "$CI_CODECOV_TOKEN" != "" ]; then curl -s https://codecov.io/bash >codecov.sh chmod +x codecov.sh ./codecov.sh -t "$CI_CODECOV_TOKEN" -n 'rsyslog buildbot PR' &> codecov_log rm codecov.sh lines="$(wc -l < codecov_log)" if (( lines > 3000 )); then printf 'codecov log file is very large (%d lines), showing parts\n' $lines head -n 1500 < codecov_log printf '\n\n... snip ...\n\n' tail -n 1500 < codecov_log else cat codecov_log fi rm codecov_log fi exit $rc liblognorm-2.0.8/doc/000077500000000000000000000000001511425433100144315ustar00rootroot00000000000000liblognorm-2.0.8/doc/.gitignore000066400000000000000000000000341511425433100164160ustar00rootroot00000000000000_build Makefile.in Makefile liblognorm-2.0.8/doc/Makefile.am000066400000000000000000000026121511425433100164660ustar00rootroot00000000000000EXTRA_DIST = _static _templates conf.py \ index.rst introduction.rst installation.rst \ configuration.rst sample_rulebase.rst internals.rst \ contacts.rst changes.rst libraryapi.rst \ lognormalizer.rst license.rst graph.png htmldir = $(docdir) built_html = _build/html #html_DATA = $(built_html)/index.html # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = -n -W -c $(srcdir) #SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) $(srcdir) .PHONY: clean-local html-local man-local all-local dist-hook install-data-hook dist-hook: find $(distdir)/ -name .gitignore | xargs rm -f clean-local: -rm -rf $(BUILDDIR)/* html-local: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." man-local: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." all-local: html-local install-data-hook: find $(built_html) -type f -printf "%P\n" | \ while read file; do \ echo " $(INSTALL_DATA) -D $(built_html)/$$file '$(DESTDIR)$(htmldir)/$$file'"; \ $(INSTALL_DATA) -D $(built_html)/$$file "$(DESTDIR)$(htmldir)/$$file" || exit $$?; \ done uninstall-local: -rm -rf "$(DESTDIR)$(htmldir)" liblognorm-2.0.8/doc/Makefile.sphinx000066400000000000000000000127341511425433100174100ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Testprojectlpk.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Testprojectlpk.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/Testprojectlpk" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Testprojectlpk" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." liblognorm-2.0.8/doc/_static/000077500000000000000000000000001511425433100160575ustar00rootroot00000000000000liblognorm-2.0.8/doc/_static/.gitignore000066400000000000000000000000001511425433100200350ustar00rootroot00000000000000liblognorm-2.0.8/doc/_templates/000077500000000000000000000000001511425433100165665ustar00rootroot00000000000000liblognorm-2.0.8/doc/_templates/.gitignore000066400000000000000000000000001511425433100205440ustar00rootroot00000000000000liblognorm-2.0.8/doc/changes.rst000066400000000000000000000001311511425433100165660ustar00rootroot00000000000000ChangeLog ========= See below for a list of changes. .. literalinclude:: ../ChangeLog liblognorm-2.0.8/doc/conf.py000066400000000000000000000172261511425433100157400ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Liblognorm documentation build configuration file, created by # sphinx-quickstart on Mon Dec 16 13:12:44 2013. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # General information about the project. project = u'Liblognorm' # pylint: disable=W0141 copyright = u'Adiscon' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '1.1' # The full version, including alpha/beta/rc tags. release = '1.1.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'haiku' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None html_title = "A fast log normalization library" # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None html_short_title = project + " " + release + " documentation" # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. html_use_index = False # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. html_show_copyright = False # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'Liblognormdoc' # -- Options for LaTeX output -------------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'Liblognorm.tex', u'Liblognorm Documentation', u'Pavel Levshin', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'liblognorm', u'Liblognorm Documentation', [u'Pavel Levshin'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'Liblognorm', u'Liblognorm Documentation', u'Pavel Levshin', 'Liblognorm', 'Fast log normalization library.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' liblognorm-2.0.8/doc/configuration.rst000066400000000000000000001274261511425433100200460ustar00rootroot00000000000000How to configure ================ To use liblognorm, you need 3 things. 1. An installed and working copy of liblognorm. The installation process has been discussed in the chapter :doc:`installation`. 2. Log files. 3. A rulebase, which is heart of liblognorm configuration. Log files --------- A log file is a text file, which typically holds many lines. Each line is a log message. These are usually a bit strange to read, thus to analyze. This mostly happens, if you have a lot of different devices, that are all creating log messages in a different format. Rulebase -------- The rulebase holds all the schemes for your logs. It basically consists of many lines that reflect the structure of your log messages. When the normalization process is started, a parse-tree will be generated from the rulebase and put into the memory. This will then be used to parse the log messages. Each line in rulebase file is evaluated separately. Rulebase Versions ----------------- This documentation is for liblognorm version 2 and above. Version 2 is a complete rewrite of liblognorm which offers many enhanced features but is incompatible to some pre-v2 rulebase commands. For details, see compatibility document. Note that liblognorm v2 contains a full copy of the v1 engine. As such it is fully compatible to old rulebases. In order to use the new v2 engine, you need to explicitly opt in. To do so, you need to add the line:: version=2 to the top of your rulebase file. Currently, it is very important that * the line is given exactly as above * no whitespace within the sequence is permitted (e.g. "version = 2" is invalid) * no whitespace or comment after the "2" is permitted (e.g. "version=2 # comment") is invalid * this line **must** be the **very** first line of the file; this also means there **must** not be any comment or empty lines in front of it Only if the version indicator is properly detected, the v2 engine is used. Otherwise, the v1 engine is used. So if you use v2 features but got the version line wrong, you'll end up with error messages from the v1 engine. The v2 engine understands almost all v1 parsers, and most importantly all that are typically used. It does not understand these parsers: * tokenized * recursive * descent * regex * interpret * suffixed * named_suffixed The recursive and descent parsers should be replaced by user-defined types in. The tokenized parsers should be replaced by repeat. The interpret functionality is provided via the parser's "format" parameters. For the others, currently there exists no replacement, but will the exception of regex, will be added based on demand. If you think regex support is urgently needed, please read our `related issue on github, `_ where you can also cast you ballot in favor of it. If you need any of these parsers, you need to use the v1 engine. That of course means you cannot use the v2 enhancements, so converting as much as possible makes sense. Commentaries ------------ To keep your rulebase tidy, you can use commentaries. Start a commentary with "#" like in many other configurations. It should look like this:: # The following prefix and rules are for firewall logs Note that the comment character MUST be in the first column of the line. Empty lines are just skipped, they can be inserted for readability. User-Defined Types ------------------ If the line starts with ``type=``, then it contains a user-defined type. You can use a user-defined type wherever you use a built-in type; they are equivalent. That also means you can use user-defined types in the definition of other user-defined types (they can be used recursively). The only restriction is that you must define a type **before** you can use it. This line has following format:: type=: Everything before the colon is treated as the type name. User-defined types must always start with "@". So "@mytype" is a valid name, whereas "mytype" is invalid and will lead to an error. After the colon, a match description should be given. It is exactly the same like the one given in rule lines (see below). A generic IP address type could look as follows:: type=@IPaddr:%ip:ipv4% type=@IPaddr:%ip:ipv6% This creates a type "@IPaddr", which consists of either an IPv4 or IPv6 address. Note how we use two different lines to create an alternative representation. This is how things generally work with types: you can use as many "type" lines for a single type as you need to define your object. Note that pure alternatives could also be defined via the "alternative" parser - which option to choose is left to the user. They are equivalent. The ability to use multiple type lines for definition, however, brings more power than just to define alternatives. Includes -------- Especially with user-defined types includes come handy. With an include, you can include definitions already made elsewhere into the current rule set (just like the "include" directive works in many programming languages). An include is done by a line starting with ``include=`` where the rest of the line is the actual file name, just like in this example:: include=/var/lib/liblognorm/stdtypes.rb The definition is included right at the position where it occurs. Processing of the original file is continued when the included file has been fully processed. Includes can be nested. To facilitate repositories of common rules, liblognorm honors the :: LIBLOGNORM_RULEBASES environment variable. If it is set liblognorm tries to locate the file inside the path pointed to by ``LIBLOGNORM_RULEBASES`` in the following case: * the provided file cannot be found * the provided file name is not an absolute path (does not start with "/") So assuming we have:: export LIBLOGNORM_RULEBASES=/var/lib/loblognorm The above example can be re-written as follows:: include=stdtypes.rb Note, however, that if ``stdtypes.rb`` exist in the current working directory, that file will be loaded insted of the one from ``/var/lib/liblognorm``. This use facilitates building a library of standard type definitions. Note the the liblognorm project also ships type definitions for common scenarios. Rules ----- If the line starts with ``rule=``, then it contains a rule. This line has following format:: rule=[[,...]]: Everything before a colon is treated as comma-separated list of tags, which will be attached to a match. After the colon, match description should be given. It consists of string literals and field selectors. String literals should match exactly, whereas field selectors may match variable parts of a message. A rule could look like this (in legacy format):: rule=:%date:date-rfc3164% %host:word% %tag:char-to:\x3a%: no longer listening on %ip:ipv4%#%port:number%' This excerpt is a common rule. A rule always contains several different "parts"/properties and reflects the structure of the message you want to normalize (e.g. Host, IP, Source, Syslogtag...). Literals -------- Literal is just a sequence of characters, which must match exactly. Percent sign characters must be escaped to prevent them from starting a field accidentally. Replace each "%" with "\\x25" or "%%", when it occurs in a string literal. Fields ------ There are different formats for field specification: * legacy format * condensed format * full json format Legacy Format ############# Legay format is exactly identical to the v1 engine. This permits you to use existing v1 rulebases without any modification with the v2 engine, except for adding the ``version=2`` header line to the top of the file. Remember: some v1 types are not supported - if you are among the few who use them, you need to do some manual conversion. For almost all users, manual conversion should not be necessary. Legacy format is not documented here. If you want to use it, see the v1 documentation. Condensed Format ################ The goal of this format is to be as brief as possible, permitting you an as-clear-as-possible view of your rule. It is very similar to legacy format and recommended to be used for simple types which do not need any parser parameters. Its structure is as follows:: %:{}% **field name** -> that name can be selected freely. It should be a description of what kind of information the field is holding, e.g. SRC is the field contains the source IP address of the message. These names should also be chosen carefully, since the field name can be used in every rule and therefore should fit for the same kind of information in different rules. Some special field names exist: * **dash** ("-"): this field is matched but not saved * **dot** ("."): this is useful if a parser returns a set of fields. Usually, it does so by creating a json subtree. If the field is named ".", then no subtree is created but instead the subfields are moved into the main hierarchy. * **two dots** (".."): similar to ".", but can be used at the lower level to denote that a field is to be included with the name given by the upper-level object. Note that ".." is only acted on if a subelement contains a single field. The reason is that if there were more, we could not assign all of them to the *single* name given by the upper-level-object. The prime use case for this special name is in user-defined types that parse only a single value. Without "..", they would always become a JSON subtree, which seems unnatural and is different from built-in types. So it is suggested to name such fields as "..", which means that the user can assign a name of his liking, just like in the case of built-in parsers. **field type** -> selects the accordant parser, which are described below. Special characters that need to be escaped when used inside a field description are "%" and ":". It is strongly recommended **not** to use them. **parameters** -> This is an optional set of parameters, given in pure JSON format. Parameters can be generic (e.g. "priority") or specific to a parser (e.g. "extradata"). Generic parameters are described below in their own section, parser-specific ones in the relevant type documentation. As an example, the "char-to" parser accepts a parameter named "extradata" which describes up to which character it shall match (the name "extradata" stems back to the legacy v1 system):: %tag:char-to{"extradata":":"}% Whitespace, including LF, is permitted inside a field definition after the opening percent sign and before the closing one. This can be used to make complex rules more readable. So the example rule from the overview section above could be rewritten as:: rule=:% date:date-rfc3164 % % host:word % % tag:char-to{"extradata":":"} %: no longer listening on % ip:ipv4 %#% port:number %' When doing this, note well that whitespace IS important inside the literal text. So e.g. in the second example line above "% %" we require a single SP as literal text. Note that any combination of your liking is valid, so it could also be written as:: rule=:%date:date-rfc3164% %host:word% % tag:char-to{"extradata":":"} %: no longer listening on % ip:ipv4 %#% port:number %' To prevent a typical user error, continuation lines are **not** permitted to start with ``rule=``. There are some obscure cases where this could be a valid rule, and it can be re-formatted in that case. Moreoften, this is the result of a missing percent sign, as in this sample:: rule=:test%field:word ... missing percent sign ... rule=:%f:word% If we would permit ``rule=`` at start of continuation line, these kinds of problems would be very hard to detect. Full JSON Format ################ This format is best for complex definitions or if there are many parser parameters. Its structure is as follows:: %JSON% Where JSON is the configuration expressed in JSON. To get you started, let's rewrite above sample in pure JSON form:: rule=:%[ {"type":"date-rfc3164", "name":"date"}, {"type":"literal", "text:" "}, {"type":"char-to", "name":"host", "extradata":":"}, {"type":"literal", "text:": no longer listening on "}, {"type":"ipv4", "name":"ip"}, {"type":"literal", "text:"#"}, {"type":"number", "name":"port"} ]% A couple of things to note: * we express everything in this example in a *single* parser definition * this is done by using a **JSON array**; whenever an array is used, multiple parsers can be specified. They are executed one after the other in given order. * literal text is matched here via explicit parser call; as specified below, this is recommended only for specific use cases with the current version of liblognorm * parser parameters (both generic and parser-specific ones) are given on the main JSON level * the literal text shall not be stored inside an output variable; for this reason no name attribute is given (we could also have used ``"name":"-"`` which achieves the same effect but is more verbose). With the literal parser calls replaced by actual literals, the sample looks like this:: rule=:%{"type":"date-rfc3164", "name":"date"} % % {"type":"char-to", "name":"host", "extradata":":"} % no longer listening on % {"type":"ipv4", "name":"ip"} %#% {"type":"number", "name":"port"} % Which format you use and how you exactly use it is up to you. Some guidelines: * using the "literal" parser in JSON should be avoided currently; the experimental version does have some rough edges where conflicts in literal processing will not be properly handled. This should not be an issue in "closed environments", like "repeat", where no such conflict can occur. * otherwise, JSON is perfect for very complex things (like nesting of parsers - it is **not** suggested to use any other format for these kinds of things. * if a field needs to be matched but the result of that match is not needed, omit the "name" attribute; specifically avoid using the more verbose ``"name":"-"``. * it is a good idea to start each definition with ``"type":"..."`` as this provides a good quick overview over what is being defined. Mandatory Parameters .................... type ~~~~ The field type, selects the parser to use. See "fields" below for description. Optional Generic Parameters ........................... name ~~~~ The field name to use. If "-" is used, the field is matched, but not stored. In this case, you can simply **not** specify a field name, which is the preferred way of doing this. priority ~~~~~~~~ The priority to assign to this parser. Priorities are numerical values in the range from 0 (highest) to 65535 (lowest). If multiple parsers could match at a given character position of a log line, parsers are tried in priority order. Different priorities can lead to different parsing. For example, if the greedy "rest" type is assigned priority 0, and no other parser is assigned the same priority, no other parser will ever match (because "rest" is very greedy and always matches the rest of the message). Note that liblognorm internally has a parser-specific priority, which is selected by the program developer based on the specificality of a type. If the user assigns equal priorities, parsers are executed based on the parser-specific priority. The default priority value is 30,000. Field types ----------- We have legacy and regular field types. Pre-v2, we did not have user-defined types. As such, there was a relatively large number of parsers that handled very similar cases, for example for strings. These parsers still work and may even provide best performance in extreme cases. In v2, we focus on fewer, but more generic parsers, which are then tailored via parameters. There is nothing bad about using legacy parsers and there is no plan to outphase them at any time in the future. We just wanted to let you know, especially if you wonder about some "wereid" parsers. In v1, parsers could have only a single parameter, which was called "extradata" at that time. This is why some of the legacy parsers require or support a parameter named "extradata" and do not use a better name for it (internally, the legacy format creates a v2 parser definition with "extradata" being populated from the legacy "extradata" part of the configuration). number ###### One or more decimal digits. Parameters .......... format ~~~~~~ Specifies the format of the json object. Possible values are "string" and "number", with string being the default. If "number" is used, the json object will be a native json integer. maxval ~~~~~~ Maximum value permitted for this number. If the value is higher than this, it will not be detected by this parser definition and an alternate detection path will be pursued. float ##### A floating-pt number represented in non-scientific form. Parameters .......... format ~~~~~~ Specifies the format of the json object. Possible values are "string" and "number", with string being the default. If "number" is used, the json object will be a native json floating point number. Note that we try to preserve the original string serialization format, but keep on your mind that floating point numbers are inherently imprecise, so slight variance may occur depending on processing them. hexnumber ######### A hexadecimal number as seen by this parser begins with the string "0x", is followed by 1 or more hex digits and is terminated by white space. Any interleaving non-hex digits will cause non-detection. The rules are strict to avoid false positives. Parameters .......... format ~~~~~~ Specifies the format of the json object. Possible values are "string" and "number", with string being the default. If "number" is used, the json object will be a native json integer. Note that json numbers are always decimal, so if "number" is selected, the hex number will be converted to decimal. The original hex string is no longer available in this case. maxval ~~~~~~ Maximum value permitted for this number. If the value is higher than this, it will not be detected by this parser definition and an alternate detection path will be pursued. This is most useful if fixed-size hex numbers need to be processed. For example, for byte values the "maxval" could be set to 255, which ensures that invalid values are not misdetected. kernel-timestamp ################ Parses a linux kernel timestamp, which has the format:: [ddddd.dddddd] where "d" is a decimal digit. The part before the period has to have at least 5 digits as per kernel code. There is no upper limit per se inside the kernel, but liblognorm does not accept more than 12 digits, which seems more than sufficient (we may reduce the max count if misdetections occur). The part after the period has to have exactly 6 digits. whitespace ########## This parses all whitespace until the first non-whitespace character is found. This check is performed using the ``isspace()`` C library function to check for space, horizontal tab, newline, vertical tab, feed and carriage return characters. This parser is primarily a tool to skip to the next "word" if the exact number of whitespace characters (and type of whitespace) is not known. The current parsing position MUST be on a whitespace, else the parser does not match. Remember that to just parse but not preserve the field contents, the dash ("-") is used as field name in compact format or the "name" parameter is simply omitted in JSON format. This is almost always expected with the *whitespace* type. string ###### This is a highly customizable parser that can be used to extract many types of strings. It is meant to be used for most cases. It is suggested that specific string types are created as user-defined types using this parser. This parser supports: * various quoting modes for strings * escape character processing Parameters .......... quoting.mode ~~~~~~~~~~~~ Specifies how the string is quoted. Possible modes: * **none** - no quoting is permitted * **required** - quotes must be present * **auto** - quotes are permitted, but not required Default is ``auto``. quoting.escape.mode ~~~~~~~~~~~~~~~~~~~ Specifies how quote character escaping is handled. Possible modes: * **none** - there are no escapes, quote characters are *not* permitted in value * **double** - the ending quote character is duplicated to indicate a single quote without termination of the value (e.g. ``""``) * **backslash** - a backslash is prepended to the quote character (e.g ``\"``) * **both** - both double and backslash escaping can happen and are supported Default is ``both``. Note that turning on ``backslash`` mode (or ``both``) has the side-effect that backslash escaping is enabled in general. This usually is what you want if this option is selected (e.g. otherwise you could no longer represent backslash). **NOTE**: this parameter also affects operation if quoting is **turned off**. That is somewhat counter-intuitive, but has traditionally been the case - which means we cannot change it. quoting.char.begin ~~~~~~~~~~~~~~~~~~ Sets the begin quote character. Default is ". quoting.char.end ~~~~~~~~~~~~~~~~ Sets the end quote character. Default is ". Note that setting the begin and end quote character permits you to support more quoting modes. For example, brackets and braces are used by some software for quoting. To handle such string, you can for example use a configuration like this:: rule=:a %f:string{"quoting.char.begin":"[", "quoting.char.end":"]"}% b which matches strings like this:: a [test test2] b matching.permitted ~~~~~~~~~~~~~~~~~~ This allows to specify a set of characters permitted in the to-be-parsed field. It is primarily a utility to extract things like programming-language like names (e.g. consisting of letters, digits and a set of special characters only), alphanumeric or alphabetic strings. If this parameter is not specified, all characters are permitted. If it is specified, only the configured characters are permitted. Note that this option reliably only works on US-ASCII data. Multi-byte character encodings may lead to strange results. There are two ways to specify permitted characters. The simple one is to specify them directly for the parameter:: rule=:%f:string{"matching.permitted":"abc"}% This only supports literal characters and all must be given as a single parameter. For more advanced use cases, an array of permitted characters can be provided:: rule=:%f:string{"matching.permitted":[ {"class":"digit"}, {"chars":"xX"} ]}% Here, ``class`` is a specify for the usual character classes, with support for: * digit * hexdigit * alpha * alnum In contrast, ``chars`` permits to specify literal characters. Both ``class`` as well as ``chars`` may be specified multiple times inside the array. For example, the ``alnum`` class could also be permitted as follows:: rule=:%f:string{"matching.permitted":[ {"class":"digit"}, {"class":"alpha"} ]}% matching.mode ~~~~~~~~~~~~~ This parameter permits the strict matching requirement of liblognorm, where each parser must be terminated by a space character. Possible values are: * **strict** - which requires that space * **lazy** - which does not Default is ``strict``, this parameter is available starting with version 2.0.6. In ``lazy`` mode, the parser always matches if at least one character can be matched. This can lead to unexpected results, so use it with care. Example: assume the following message (without quotes):: "12:34 56" And the following parser definition:: rule=:%f:string{"matching.permitted":[ {"class":"digit"} ]} %%r:rest% This will be unresolvable, as ":" is not a digit. With this definition:: rule=:%f:string{"matching.permitted":[ {"class":"digit"} ], "matching.mode":"lazy"} %%r:rest% it becomes resolvable, and ``f`` will contain "12" and ``r`` will contain ":34 56". This also shows the risk associated, as the result obtained may not necessarily be what was intended. word #### One or more characters, up to the next space (\\x20), or up to end of line. string-to ######### One or more characters, up to the next string given in "extradata". alpha ##### One or more alphabetic characters, up to the next whitespace, punctuation, decimal digit or control character. char-to ####### One or more characters, up to the next character(s) given in extradata. Parameters .......... extradata ~~~~~~~~~ This is a mandatory parameter. It contains one or more characters, each of which terminates the match. char-sep ######## Zero or more characters, up to the next character(s) given in extradata. Parameters .......... extradata ~~~~~~~~~~ This is a mandatory parameter. It contains one or more characters, each of which terminates the match. rest #### Zero or more characters until end of line. Must always be at end of the rule, even though this condition is currently **not** checked. In any case, any definitions after *rest* are ignored. Note that the *rest* syntax should be avoided because it generates a very broad match. If it needs to be used, the user shall assign it the lowest priority among his parser definitions. Note that the parser-specific priority is also lowest, so by default it will only match if nothing else matches. quoted-string ############# Zero or more characters, surrounded by double quote marks. Quote marks are stripped from the match. op-quoted-string ################ Zero or more characters, possibly surrounded by double quote marks. If the first character is a quote mark, operates like quoted-string. Otherwise, operates like "word" Quote marks are stripped from the match. date-iso ######## Date in ISO format ('YYYY-MM-DD'). time-24hr ######### Time of format 'HH:MM:SS', where HH is 00..23. time-12hr ######### Time of format 'HH:MM:SS', where HH is 00..12. duration ######## A duration is similar to a timestamp, except that it tells about time elapsed. As such, hours can be larger than 23 and hours may also be specified by a single digit (this, for example, is commonly done in Cisco software). Examples for durations are "12:05:01", "0:00:01" and "37:59:59" but not "00:60:00" (HH and MM must still be within the usual range for minutes and seconds). date-rfc3164 ############ Valid date/time in RFC3164 format, i.e.: 'Oct 29 09:47:08'. This parser implements several quirks to match malformed timestamps from some devices. Parameters .......... format ~~~~~~ Specifies the format of the json object. Possible values are - **string** - string representation as given in input data - **timestamp-unix** - string converted to an unix timestamp (seconds since epoch) - **timestamp-unix-ms** - a kind of unix-timestamp, but with millisecond resolution. This format is understood for example by ElasticSearch. Note that RFC3164 does **not** contain subsecond resolution, so this option makes no sense for RFC3164-data only. It is useful, however, if processing mixed sources, some of which contain higher precision. date-rfc5424 ############ Valid date/time in RFC5424 format, i.e.: '1985-04-12T19:20:50.52-04:00'. Slightly different formats are allowed. Parameters .......... format ~~~~~~ Specifies the format of the json object. Possible values are - **string** - string representation as given in input data - **timestamp-unix** - string converted to an unix timestamp (seconds since epoch). If subsecond resolution is given in the original timestamp, it is lost. - **timestamp-unix-ms** - a kind of unix-timestamp, but with millisecond resolution. This format is understood for example by ElasticSearch. Note that a RFC5424 timestamp can contain higher than ms resolution. If so, the timestamp is truncated to millisecond resolution. ipv4 #### IPv4 address, in dot-decimal notation (AAA.BBB.CCC.DDD). ipv6 #### IPv6 address, in textual notation as specified in RFC4291. All formats specified in section 2.2 are supported, including embedded IPv4 address (e.g. "::13.1.68.3"). Note that a **pure** IPv4 address ("13.1.68.3") is **not** valid and as such not recognized. To avoid false positives, there must be either a whitespace character after the IPv6 address or the end of string must be reached. mac48 ##### The standard (IEEE 802) format for printing MAC-48 addresses in human-friendly form is six groups of two hexadecimal digits, separated by hyphens (-) or colons (:), in transmission order (e.g. 01-23-45-67-89-ab or 01:23:45:67:89:ab ). This form is also commonly used for EUI-64. from: http://en.wikipedia.org/wiki/MAC_address cef ### This parses ArcSight Comment Event Format (CEF) as described in the "Implementing ArcSight CEF" manual revision 20 (2013-06-15). It matches a format that closely follows the spec. The header fields are extracted into the field name container, all extension are extracted into a container called "Extensions" beneath it. Example ....... Rule (compact format):: rule=:%f:cef' Data:: CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a value cc=field 3 Result:: { "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a value", "cc": "field 3" } } } checkpoint-lea ############## This supports the LEA on-disk format. Unfortunately, the format is underdocumented, the Checkpoint docs we could get hold of just describe the API and provide a field dictionary. In a nutshell, what we do is extract field names up to the colon and values up to the semicolon. No escaping rules are known to us, so we assume none exists (and as such no semicolon can be part of a value). This format needs to continue until the end of the log message. We have also seen some samples of a LEA format that has data **after** the format described above. So it does not end at the end of log line. We guess that this is LEA when used inside (syslog) messages. We have one sample where the format ends on a brace (`; ]`). To support this, the `terminator` parameter exists (see below). If someone has a definitive reference or a sample set to contribute to the project, please let us know and we will check if we need to add additional transformations. Parameters .......... terminator ~~~~~~~~~~ Must be a single character. If used, LEA format is terminated when the character is hit instead of a field name. Note that the terminator character is **not** part of LEA. It it should be skipped, it must be specified as a literal after the parser. We have implemented it in this way as this provides most options for this format - about which we do not know any details. Example ....... This configures a LEA parser for use with the syslog transfer format (if we guess right). It terminates when a brace is detected. Rule (condensed format):: rule=:%field:checkpoint-lea{"terminator": "]"}%] Data:: tcp_flags: RST-ACK; src: 192.168.0.1; ] Result:: { "field": { "tcp_flags": "RST-ACK", "src": "192.168.0.1" } }' cisco-interface-spec #################### A Cisco interface specifier, as for example seen in PIX or ASA. The format contains a number of optional parts and is described as follows (in ABNF-like manner where square brackets indicate optional parts): :: [interface:]ip/port [SP (ip2/port2)] [[SP](username)] Samples for such a spec are: * outside:192.168.52.102/50349 * inside:192.168.1.15/56543 (192.168.1.112/54543) * outside:192.168.1.13/50179 (192.168.1.13/50179)(LOCAL\some.user) * outside:192.168.1.25/41850(LOCAL\RG-867G8-DEL88D879BBFFC8) * inside:192.168.1.25/53 (192.168.1.25/53) (some.user) * 192.168.1.15/0(LOCAL\RG-867G8-DEL88D879BBFFC8) Note that the current version of liblognorm does not permit sole IP addresses to be detected as a Cisco interface spec. However, we are reviewing more Cisco message and need to decide if this is to be supported. The problem here is that this would create a much broader parser which would potentially match many things that are **not** Cisco interface specs. As this object extracts multiple subelements, it create a JSON structure. Let's for example look at this definition (compact format):: %ifaddr:cisco-interface-spec% and assume the following message is to be parsed:: outside:192.168.1.13/50179 (192.168.1.13/50179) (LOCAL\some.user) Then the resulting JSON will be as follows:: { "ifaddr": { "interface": "outside", "ip": "192.168.1.13", "port": "50179", "ip2": "192.168.1.13", "port2": "50179", "user": "LOCAL\\some.user" } } Subcomponents that are not given in the to-be-normalized string are also not present in the resulting JSON. iptables ######## Name=value pairs, separated by spaces, as in Netfilter log messages. Name of the selector is not used; names from the line are used instead. This selector always matches everything till end of the line. Cannot match zero characters. cisco-interface-spec #################### This is an experimental parser. It is used to detect Cisco Interface Specifications. A sample of them is: :: outside:176.97.252.102/50349 Note that this parser does not yet extract the individual parts due to the restrictions in current liblognorm. This is planned for after a general algorithm overhaul. In order to match, this syntax must start on a non-whitespace char other than colon. json #### This parses native JSON from the message. All data up to the first non-JSON is parsed into the field. There may be any other field after the JSON, including another JSON section. Note that any white space after the actual JSON is considered **to be part of the JSON**. So you cannot filter on whitespace after the JSON. Example ....... Rule (compact format):: rule=:%field1:json%interim text %field2:json%' Data:: {"f1": "1"} interim text {"f2": 2} Result:: { "field2": { "f2": 2 }, "field1": { "f1": "1" } } Note also that the space before "interim" must **not** be given in the rule, as it is consumed by the JSON parser. However, the space after "text" is required. alternative ########### This type permits to specify alternative ways of parsing within a single definition. This can make writing rule bases easier. It also permits the v2 engine to create a more efficient parsing data structure resulting in better performance (to be noticed only in extreme cases, though). An example explains this parser best:: rule=:a % {"type":"alternative", "parser": [ {"name":"num", "type":"number"}, {"name":"hex", "type":"hexnumber"} ] }% b This rule matches messages like these:: a 1234 b a 0xff b Note that the "parser" parameter here needs to be provided with an array of *alternatives*. In this case, the JSON array is **not** interpreted as a sequence. Note, though that you can nest definitions by using custom types. repeat ###### This parser is used to extract a repeated sequence with the same pattern. An example explains this parser best:: rule=:a % {"name":"numbers", "type":"repeat", "parser":[ {"type":"number", "name":"n1"}, {"type":"literal", "text":":"}, {"type":"number", "name":"n2"} ], "while":[ {"type":"literal", "text":", "} ] }% b This matches lines like this:: a 1:2, 3:4, 5:6, 7:8 b and will generate this JSON:: { "numbers": [ { "n2": "2", "n1": "1" }, { "n2": "4", "n1": "3" }, { "n2": "6", "n1": "5" }, { "n2": "8", "n1": "7" } ] } As can be seen, there are two parameters to "alternative". The parser parameter specifies which type should be repeatedly parsed out of the input data. We could use a single parser for that, but in the example above we parse a sequence. Note the nested array in the "parser" parameter. If we just wanted to match a single list of numbers like:: a 1, 2, 3, 4 b we could use this definition:: rule=:a % {"name":"numbers", "type":"repeat", "parser": {"type":"number", "name":"n"}, "while": {"type":"literal", "text":", "} }% b Note that in this example we also removed the redundant single-element array in "while". The "while" parameter tells "repeat" how long to do repeat processing. It is specified by any parser, including a nested sequence of parser (array). As long as the "while" part matches, the repetition is continued. If it no longer matches, "repeat" processing is successfully completed. Note that the "parser" parameter **must** match at least once, otherwise "repeat" fails. In the above sample, "while" mismatches after "4", because no ", " follows. Then, the parser terminates, and according to definition the literal " b" is matched, which will result in a successful rule match (note: the "a ", " b" literals are just here for explanatory purposes and could be any other rule element). Sometimes we need to deal with malformed messages. For example, we could have a sequence like this:: a 1:2, 3:4,5:6, 7:8 b Note the missing space after "4,". To handle such cases, we can nest the "alternative" parser inside "while":: rule=:a % {"name":"numbers", "type":"repeat", "parser":[ {"type":"number", "name":"n1"}, {"type":"literal", "text":":"}, {"type":"number", "name":"n2"} ], "while": { "type":"alternative", "parser": [ {"type":"literal", "text":", "}, {"type":"literal", "text":","} ] } }% b This definition handles numbers being delimited by either ", " or ",". For people with programming skills, the "repeat" parser is described by this pseudocode:: do parse via parsers given in "parser" if parsing fails abort "repeat" unsuccessful parse via parsers given in "while" while the "while" parsers parsed successfully if not aborted, flag "repeat" as successful Parameters .......... option.permitMismatchInParser ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If set to "True", permits repeat to accept as successful even when the parser processing failed. This by default is false, and can be set to true to cover some border cases, where the while part cannot definitely detect the end of processing. An example of such a border case is a listing of flags, being terminated by a double space where each flag is delimited by single spaces. For example, Cisco products generate such messages (note the flags part):: Aug 18 13:18:45 192.168.0.1 %ASA-6-106015: Deny TCP (no connection) from 10.252.88.66/443 to 10.79.249.222/52746 flags RST on interface outside cee-syslog ########## This parses cee syslog from the message. This format has been defined by Mitre CEE as well as Project Lumberjack. This format essentially is JSON with additional restrictions: * The message must start with "@cee:" * an JSON **object** must immediately follow (whitespace before it permitted, but a JSON array is **not** permitted) * after the JSON, there must be no other non-whitespace characters. In other words: the message must consist of a single JSON object only, prefixed by the "@cee:" cookie. Note that the cee cookie is case sensitive, so "@CEE:" is **NOT** valid. Prefixes -------- Several rules can have a common prefix. You can set it once with this syntax:: prefix= Prefix match description syntax is the same as rule match description. Every following rule will be treated as an addition to this prefix. Prefix can be reset to default (empty value) by the line:: prefix= You can define a prefix for devices that produce the same header in each message. We assume, that you have your rules sorted by device. In such a case you can take the header of the rules and use it with the prefix variable. Here is a example of a rule for IPTables (legacy format, to be converted later):: prefix=%date:date-rfc3164% %host:word% %tag:char-to:-\x3a%: rule=:INBOUND%INBOUND:char-to:-\x3a%: IN=%IN:word% PHYSIN=%PHYSIN:word% OUT=%OUT:word% PHYSOUT=%PHYSOUT:word% SRC=%source:ipv4% DST=%destination:ipv4% LEN=%LEN:number% TOS=%TOS:char-to: % PREC=%PREC:word% TTL=%TTL:number% ID=%ID:number% DF PROTO=%PROTO:word% SPT=%SPT:number% DPT=%DPT:number% WINDOW=%WINDOW:number% RES=0x00 ACK SYN URGP=%URGP:number% Usually, every rule would hold what is defined in the prefix at its beginning. But since we can define the prefix, we can save that work in every line and just make the rules for the log lines. This saves us a lot of work and even saves space. In a rulebase you can use multiple prefixes obviously. The prefix will be used for the following rules. If then another prefix is set, the first one will be erased, and new one will be used for the following rules. Rule tags --------- Rule tagging capability permits very easy classification of syslog messages and log records in general. So you can not only extract data from your various log source, you can also classify events, for example, as being a "login", a "logout" or a firewall "denied access". This makes it very easy to look at specific subsets of messages and process them in ways specific to the information being conveyed. To see how it works, let’s first define what a tag is: A tag is a simple alphanumeric string that identifies a specific type of object, action, status, etc. For example, we can have object tags for firewalls and servers. For simplicity, let’s call them "firewall" and "server". Then, we can have action tags like "login", "logout" and "connectionOpen". Status tags could include "success" or "fail", among others. Tags form a flat space, there is no inherent relationship between them (but this may be added later on top of the current implementation). Think of tags like the tag cloud in a blogging system. Tags can be defined for any reason and need. A single event can be associated with as many tags as required. Assigning tags to messages is simple. A rule contains both the sample of the message (including the extracted fields) as well as the tags. Have a look at this sample:: rule=:sshd[%pid:number%]: Invalid user %user:word% from %src-ip:ipv4% Here, we have a rule that shows an invalid ssh login request. The various fields are used to extract information into a well-defined structure. Have you ever wondered why every rule starts with a colon? Now, here is the answer: the colon separates the tag part from the actual sample part. Now, you can create a rule like this:: rule=ssh,user,login,fail:sshd[%pid:number%]: Invalid user %user:word% from %src-ip:ipv4% Note the "ssh,user,login,fail" part in front of the colon. These are the four tags the user has decided to assign to this event. What now happens is that the normalizer does not only extract the information from the message if it finds a match, but it also adds the tags as metadata. Once normalization is done, one can not only query the individual fields, but also query if a specific tag is associated with this event. For example, to find all ssh-related events (provided the rules are built that way), you can normalize a large log and select only that subset of the normalized log that contains the tag "ssh". Log annotations --------------- In short, annotations allow to add arbitrary attributes to a parsed message, depending on rule tags. Values of these attributes are fixed, they cannot be derived from variable fields. Syntax is as following:: annotate=:+="" Field value should always be enclosed in double quote marks. There can be multiple annotations for the same tag. Examples -------- Look at :doc:`sample rulebase ` for configuration examples and matching log lines. Note that the examples are currently in legacy format, only. liblognorm-2.0.8/doc/contacts.rst000066400000000000000000000007731511425433100170100ustar00rootroot00000000000000Contacts ======== Mailing list ------------ If you have any questions about the library, you may mail to the rsyslog mailing list rsyslog@lists.adiscon.com. To subscribe: http://lists.adiscon.net/mailman/listinfo/rsyslog Web site -------- http://www.liblognorm.com/ Git repositories ---------------- - https://github.com/rsyslog/liblognorm.git - git://git.adiscon.com/git/liblognorm.git Authors ------- Rainer Gerhards , Adiscon GmbH His blog: https://rainer.gerhards.net/ liblognorm-2.0.8/doc/graph.png000066400000000000000000003267431511425433100162570ustar00rootroot00000000000000‰PNG  IHDR ´X¥YbKGDÿÿÿ ½§“ IDATxœìÝi¼UÁöÿëŠ2:€€¦("ƒ˜ˆ8EÎ3¢8Ò};Ë£¦iÓs[ie=ΉY™3¦æ„Sh¡(â„ ƒ¨¨L†¨ÌŠÀaÿ_tç?KÍÊsÖ–ýý~>¾p¯ÍY¿}Þ—k­]U*•J*Rƒ¢€â ‚ ‚5*:¨µµµ™?~æÏŸŸ… fÍš5Y²dIjkk³îºë¦¦¦&555iÞ¼yÚ´i“æÍ› À@_p|ðAÆ—I“&eòäÉ™2eJ^yå•ÌŸ??µµµŸù笻îºiÛ¶m¶ÝvÛtîÜ9]»vÍŽ;;Öa=P´*ßb _,¥R)ãÆËC=”Ñ£Gç™gžÉûï¿Ÿ–-[¦[·nÙn»í²õÖ[§M›6iݺu6Ùd“4oÞ< 6L“&MR]]åË—gåÊ•yÿý÷³pá¼ùæ›™7o^æÌ™“iÓ¦eÊ”)™6mZV¬X‘Ö­[§oß¾éß¿8à€´iÓ¦è_ð92ÀÄ3Ï<“áÇçÞ{ïÍìÙ³Ó±cÇì¶Ûnéׯ_vÝu×l¶ÙfŸëùjkkóâ‹/fôèÑ=zt{ì±,Z´(={öÌ¡‡š£Ž:ÊXk!”±eË–åæ›oε×^›^x!;ì°C?üðtÐAéÔ©S½¶¬Zµ*£GÎ]wÝ•»ï¾; .Ì“O>9{î¹g½¶Ÿ!”¡¥K—æç?ÿy.½ôÒ¼ÿþû4hPN>ùäì´ÓNE§%ùËsï¹çž\{íµ=ztvÜqÇüÏÿüOöÝwßTUUü „PFÖ¬Y“_þò—ùîw¿›U«VåŒ3ÎÈÙgŸ-Zö‰&L˜üà1bDzôè‘«®º*_ùÊWŠÎ>£E1~üøôêÕ+C‡Í!CòÚk¯å /,ëq0I¾üå/çÞ{ïÍøñã³îºë¦W¯^9ù䓳hÑ¢¢Ó€ÏÀ@+•JùÙÏ~–wÞ9ë®»nþô§?å§?ýiÙƒo‡vȨQ£rã7æ¾ûîË;짟~ºè,àŸ0@-Z”}öÙ'ßùÎwò£ý(?þx:wî\tÖ¿­ªª*G}t&Ožœ.]º¤oß¾¹ôÒK‹Î>E£¢ RÍ;7ûì³O/^œ±cÇ–Í|Zµj•#FäŠ+®È9眓×_=—_~y4p”!`Îœ9éÓ§Oš5k–§Ÿ~:­[·.:ésWUU•3Ï<3:tÈG‘E‹å†n0@™ñ-ÆPÏ,X]vÙ%mڴɈ#²á†Tçž}öÙ 0 GuT† Vtð7 „PV­Z•þýûgéÒ¥yòÉ'Ó´iÓ¢“êͨQ£²÷Þ{çâ‹/ΙgžYtð¿\Ûõè‚ .È믿ž‡~¸¢ÆÁ$Ù}÷Ýsýõ×ç¼óÎË„ ŠÎþ—+ žŒ;6ýû÷ÏÃ?œ=öØ£èœÂœxâ‰yâ‰'ò /duÖ):*žêA©TJÏž=³Ë.»ä²Ë.+:§Pï½÷^ºvíšÓN;-gŸ}vÑ9Pñ „Pn»í¶œqÆyå•WÒ¬Y³¢s wÇwä”SNÉ+¯¼’æÍ›Í3 |ÿûßÏСCƒÿëðÃÏÆoœ«®ºªè¨xB¨cO=õT^~ùåüñE§”ªªª|ýë_Ïõ×_Ÿ5kÖÍ@uì–[nI¿~ýÒ¾}û¢SÊÊ1Ç“7Þx#O<ñDÑ)PÑ „PÇFþýûQv6Ùd“tìØÑ@3@z÷Ýw3}úôì¼óÎE§”¥]vÙ%O=õTÑPÑ „P‡^{íµ”J¥l¹å–E§”¥/}éKyíµ×ŠÎ€Šf €:ôî»ï&IZ¶lYpIyÚxãóöÛoÍ@uhùòåI’&Mš\RžÖ[o½,]º´è ¨hB¨CÍš5Kòÿ_IÈG½óÎ;®®€‚ m´ÑFI’ \RžÞ~ûíG@1 „P‡¶ÞzëÔÔÔdòäÉE§”¥I“&eûí·/:*šêPãÆÓ£G<õÔSE§”¥±cÇfçw.:*šêØ€2räÈ”J¥¢SÊʸqãòöÛoçk_ûZÑ)PÑ „PÇŽ=öØÌœ93cÆŒ):¥¬üæ7¿I¿~ý²ÕV[­ªä?c@;øàƒ³ÞzëåÖ[o-:¥,,^¼8›o¾y~ùË_fàÀEç@E3@=˜2eJvØa‡<÷ÜséÞ½{Ñ9…ûÖ·¾•G}4Ï?ÿ|ªªªŠÎ€Šf €zrâ‰'fÊ”)yâ‰'Ò°aâs 3uêÔì¸ãŽyà²Ç{Ï@õdÁ‚éÖ­[N:é¤\pÁEçbÅŠéÙ³gºtéâvk(B¨G#GŽÌ~ûí—»îº+p@Ñ9õªT*å˜cŽÉرc3qâÄl¸á†E'ñ-ÆP¯ óÎ;/Gydž~úé¢sêÕ·¿ýíÜsÏ=¹ãŽ;ŒƒPF\Aõ¬T*åä“OÎí·ßž#F¤oß¾E'Õ©R©”oûÛ¹ôÒKsß}÷åk_ûZÑIÀßp!Ô³ªªª 6,ûï¿ ßýîwE'Õ™U«Vå„NÈe—]–[o½Õ8eÈ@hØ°anºé¦ :4GqD¾õ­oeõêÕEg}®fÍš•ÝvÛ-÷Ýw_~øávØaE'Ã@©ªªÊÅ_œë®».W]uUvÝu×Ìœ9³è¬ÏÅ]wÝ•vØ!‹/Îرc³Ûn»|!ìøãϸqã²bÅŠtéÒ%?øÁ²bÅŠ¢³þ-3gÎÌþûïŸÃ;,‡~xž{î¹têÔ©è,àS  l»í¶yî¹çòýï??ûÙÏÒµk×ÜrË-©­­-:í3Y°`AÎ;ï¼téÒ%3gĮ̂Q£ò‹_ü"ë®»nÑiÀ?a €2Q]]sÏ=7S§NMïÞ½s '¤sçιá†òÁ÷±æÌ™“óÎ;/[n¹en¸á†\tÑE™8q¢[Šà ¤ªT*•ŠŽþÑ+¯¼’ /¼0·Ýv[6Ø`ƒwÜqù¯ÿú¯ÂoÙ­­­ÍÈ‘#síµ×æÁÌFm”sÎ9'§œrJÖ[o½BÛ€ÊÜ[o½•ßüæ7ùÕ¯~•™3g¦K—.9äCrðÁ§[·niРîo zï½÷òØcå®»îÊý÷ߟwÞy'{î¹gN:é¤pÀ©®®®ó nà ¢T*åÙgŸÍÝwß»îº+3gÎL‹-Ò§OŸôíÛ7Ûo¿}ºté’M7Ýô?:Ommm^yå•Lž<9ãÇÏO<‘çŸ>kÖ¬IŸ>}r衇æÀLûöí?§OÉ@_PÓ§OÏèÑ£óä“Oæ‰'žÈìÙ³“$-[¶Ì6Ûl“M7Ý4mÛ¶Í&›l’ 6Ø 555Yo½õ²Î:ëdéÒ¥Y½zu–,Y’¥K—fÎœ9ùóŸÿœÙ³ggÆŒùàƒÒ¨Q£tëÖ-}ûöM¿~ýÒ§OŸ´jÕªàO |Þ „°–X¸pa&Ožœ©S§æ¥—^Êüùó3gΜ̟??‹/ÎÊ•+³|ùò¬\¹2Mš4Iuuuš4i’¦M›~8$¶k×.:uJçÎÓ©S§4nܸèÔ1!TáÇgðàÁñ×à¯êþiÆ@Ù2@éÞ½{~ò“Ÿ”·@s!T0!T0!T0!T0!T0!T0!T‰'æüóÏ/:(#B¨ S§NÍÅ_\tPF „PÁªJ¥R©è ~,[¶,o½õV¶ÜrË¢S€2a € æc¨`B¨`B¨`B¨`B¨`B¨`B¨ #FŒÈV[mUtPF „PA–-[–™3g”!T-ZdÇw,:(#U¥R©TtP W@3@3@3Z²dIÑ @=3@…[±bEþßÿûÙm·ÝÒ²eË¢s€zæ[Œ€¬^½:íڵ˟ÿüçü+Exýõ׳Å[Ô]Pç\AäÝwßÍøñãÿáõF¥iÓ¦ÿÒÏš3gNŽ=öØÏ+ (ˆ*Èïÿûì´ÓNÿñÏY°`AöÝwß¼õÖ[ŸCP¤FEõoÙ²eùÞ÷¾—… f“M6É|eË–}ä=S¦LÉ·¾õ­tïÞ=óæÍËĉsÅWd—]vÉ°aÃ2iÒ¤4mÚ4'Ÿ|r~ñ‹_$I–.]šË.»,³fÍÊŒ3²zõê\~ùåéÑ£Gø <ƒ*ÈÔ©Ssï½÷æÈöÛoŸk®¹&É_ž%رcǬ\¹òÃg¶oß>555™1cFJ¥RÚ¶m›u×]7¯¼òJ’¤ªª*;vÌôéÓ“$kÖ¬ÉþûïŸë®».›nºi’dРAyä‘G2sæÌ4kÖ¬€O ü3® € ²ÝvÛeÔ¨Qyúé§sÝu×}øú[l‘Í7ßöPž „Pa¶ÜrË$¹%øÓüñiÒ¤Iú÷ïŸ${õàß¾Ö¹sç,_¾<Æ ûÈ{Þ|óÍx (n1€ söÙgçÁÌYg•víÚ¥wïÞ™çœsNæΛ~ýúå7ÞȈ#rçwù‘€Oá B¨0ýû÷σ>˜víÚe·ÝvËFm”›o¾9]ºtÉI'”I“&¥¶¶6_|qª««sÀ¤Y³f9ãŒ3Ò¨Q£œp ©ªªÊ…^˜ÚÚÚüä'?I’¬¿þúyôÑG³çž{æꫯÎ1Ç“çž{.7ß|sš6mZð§>IUéÓž2 ¬Õ\Adâĉ9ÿüó‹Îʈ*ÈÔ©SsñÅ”!T0Ï € ²lÙ²¼õÖ[ÙrË-‹NÊ„*˜[Œ ‚ ‚ ‚ ‚ ‚ ‚ ‚Œ1"[mµUÑ@1@Y¶lYfÎœYtPF „PAZ´h‘wÜ±è  ŒT•J¥RÑ@1\AÌ@¬QÑ@Ýxûí·s÷Ýw¦÷n°Á9òÈ#ë¸(GžAk©•+WfuÖI’TWWâûV­Z•ã?>×_}}¥eÄ-Æ°–jܸq† ’êêê¬ZµêÿIâêA¨`® €µØ¨Q£²Ç{|ê{š7ož·Þz+yT"WÀZ¬ÿþÙh£>ñxãÆ3xð`ã T0!¬Å4hÁƒ§qãÆ{|åÊ•9ê¨£ê¹ ('n1€µÜsÏ=—ž={~ì±Í6Û,³gÏNUUU=Wå„°–ëÑ£GÚ·oÿ¯WWWç˜cŽ1@…3@8þøãS]]ý‘×V­ZåöbÀ-ÆP ¦M›–í¶Ûî#¯m»í¶™6mZAE@¹p!T€N:¥sçÎÞN\]]ãŽ;®à*  BwÜqiØ°a’dõêÕ8p`ÁE@9p‹1TˆÙ³ggóÍ7O©TÊN;í”矾è$  ¸‚*D»víÒµk×$q{1ð!WÀZdÕªUy饗2cƌ̙3'sçÎͼyó²|ùò,]º43fÌȬY³Ò·oßÔÔÔ¤iÓ¦iÚ´i6Ûl³´mÛ6íÚµËvÛm—víÚýQ€zb €/°yóæeÔ¨Qyì±Çò§?ý)S¦LÉÊ•+SUU•M7Ý4mÚ´Éf›m–u×]7M›6ÍêÕ«óøãgÏ=÷Lmmm–,Y’Å‹gÎœ9™3gN–,Y’$iÑ¢Eºuë–^½ze÷ÝwÏ.»ì’õÖ[¯àO Ô!|ÁŒ?>wÜqGî¿ÿþL›6-Mš4IŸ>}Ò½{÷tïÞ=ݺuK‡R]]ý±~æÌ™ÙrË-?öØ’%K2eÊ”Lš4)/¼ðBž}öÙLœ81ÕÕÕéÝ»w9ävØaÙtÓMëò#õÈ@_óçÏÏ/ùËÜxãyõÕW³í¶Ûæ°ÃË€Ò³gÏO?ï¼óNüñ<øàƒ¹÷Þ{³dÉ’ôíÛ7'tR9ä:=7P÷ „PÆ^|ñÅüìg?Ëí·ßž 7Ü0_ÿú×3hРl¿ýö…ô¬\¹2<òHn¼ñÆÜ{ï½Ùd“Mrꩧæ´ÓNKÓ¦M iþ3B(C3gÎÌ\[o½5]ºtÉ™gž™#<2555E§}höìÙ6lX®½öÚ4lØ0çw^N;í´¬»îºE§ÿ!”‘>ø ^xa~úÓŸfóÍ7Ïøà 80UUUE§}¢E‹å§?ýi®¼òÊ´jÕ*×\sMöÞ{€ÏÈ@eâ™gžÉ!C2{öì\tÑE9õÔSÓ¨Q£¢³>³7ß|3gŸ}v~ûÛßfðàÁ¹âŠ+Ò²eË¢³€¢AÑ@rùå—§oß¾iß¾}&Ožœ¡C‡~¡ÆÁ$iݺun»í¶<ðÀ=ztvÚi§Œ7®è,àŸ0@V¬X‘#Ž8"çž{n~ô£åá‡Î[lQtÖdß}÷ÍĉÓ©S§ìºë®¹ñÆ‹N>Åë?IÀZdÙ²e9ðÀ3yòäüáH¿~ýŠNúÜ´lÙ2<ð@~ðƒdÈ!Y¸paÎ<óÌ¢³€a €¼÷Þ{0`@fÍš•'žx"Ûn»mÑIŸ» ä‚ .H›6mrꩧfÕªU9÷Üs‹ÎþŽêYmmmŽ<òÈüùÏΓO>ù…¿¥øŸ9ñijÁäØcM›6m2xð࢓€¿a €zöo|#£GÎ3Ï<³Öƒõ×AtÈ!Ù|óÍÓ§OŸ¢“€ÿUU*•JEG@¥xôÑG³ß~ûåá‡Îî»ï^tN½;å”S2räÈLš4)Mš4):ˆêÍâÅ‹Óµk× 2$\pAÑ9…øàƒ²ÓN;¥wïÞ¹öÚk‹Îb €zóo|#=öXž}öÙTWWS˜?ýéOéÙ³gÆŒ“=zÏ@õ`îܹÙzë­óÀTä­Åï¿ÿû¿3kÖ¬Œ9²è¨x Š€Jpá…¦W¯^õ:.Z´(­ZµÊ=÷Üóo¯KßûÞ÷òøãçÉ'Ÿ¬÷se €:¶|ùòÜrË-9餓êõ¼5Ê;３%K–ü[ÇëRûöí³÷Þ{ç¿øE½Ÿø(!Ô±{ï½7 4ÈX¯çmÒ¤IÚ´i“m¶Ùæß:^׎9æ˜Ü{ï½Yºti!çþÂ@uì¾ûîË€RSSSïçîÚµkºvíúo¯Kûî»oV¯^í9„P0!Ô±§Ÿ~:½zõ*äÜguVš4iòo¯K555éÞ½{žyæ™BÎü…êÐܹs3gÎœôèÑ£ó0à?:^×vÚi§Œ7®Ð¨tB¨CóæÍK’´iÓ¦à’òÔ¾}ûÌ™3§è ¨hB¨C .L’´lÙ²à’òÔ²e˼óÎ;Eg@E3@Z¾|y’dýõ×/¸¤<­»îºY¶lYÑPÑ „P‡š5k–äÿ¿’z÷ÝwÓ¢E‹¢3 ¢ ýuüz÷Ýw .)OB(žêP‡Ò°aÃ̘1£è”²4}úôtêÔ©è ¨hB¨C믿~ºté’gžy¦è”²4nܸôèÑ£è ¨hB¨c»îºkÆŒStFÙ™7o^^yå•ôéÓ§è¨hB¨cƒ ʘ1c2{öì¢SÊÊðáóù曧wïÞE§@E3@ëÝ»w¶Øb‹ >¼è”²rË-·äè£NƒþZEò¿ÄPǪªªrÎ9ç䪫®ÊŠ+ŠÎ) <òHf̘‘ÓN;­è¨xB¨_ÿú×Ó¸qã\sÍ5E§®T*åÿþßÿ›ÓN;-­[·.:*žêAãÆsÑEå‚ .ȬY³ŠÎ)Ô/~ñ‹Ì˜1#çw^Ñ)@’ªR©T*:*Å~ûí—+VäÑGMUUUÑ9õîÕW_M÷îÝ3lØ°sÌ1Eç1@½zóÍ7Ó¥K—œxâ‰ùñ\tN½Z¼xqvÝu×l½õֹ뮻ŠÎþW£¢ ’´nÝ:ÇϾûî›Í7ß<'Ÿ|rÑIõ¢¶¶6‡zhÞÿýüæ7¿):øB¨g ÈUW]•ÓO?=믿þZ«íÊ•+3hРLš4)cÇŽMÓ¦M‹Nþ† pÊ)§dñâÅ9î¸ã²dÉ’œvÚiE'Õ‰÷Þ{/|pÆŸ‘#G¦C‡E'Ç@9ÿüó³á†æôÓOÏôéÓsÉ%—¤qãÆEg}nfÎœ™C=4óçÏÏèÑ£Ó¹s碓€Ñ è¨d§žzjî¾ûîÜtÓMéׯ_fÏž]tÒçâþûïÏŽ;î˜Få™gž1@3@Á:è <ÿüóY¶lY:w¯¾:kÖ¬):ëß²`Á‚sÌ19à€røá‡çÉ'ŸLûöí‹Î>…ÊÀ6Ûl“çŸ>gœqFÎ<óÌôë×/Ï<óLÑYŸÙêÕ«síµ×¦sçÎyüñÇ3bĈüò—¿LMMMÑiÀ?a €2QSS“‹.º(ãÇOUUUvÞyçtÐAyñÅ‹NûDkÖ¬Ém·Ý–N:eèС9ꨣ2eÊ”ì¿ÿþE§Ÿ‘ÊL·nÝòÄOäÁÌo¼‘nݺå«_ýjî¿ÿþ²¹õxÑ¢E¹ôÒKÓ¡C‡}ôÑÙyç3}úô\~ùåÙpà ‹ÎþU¥R©TtðñJ¥RFŽ™+¯¼2#GŽLûöí3hР 4(Ûo¿}½¶|ðÁyä‘GrÛm·eĈiØ°a† ’ÓN;-:t¨×àóc €/ˆ—_~97ÝtSn¿ýö¼üòËÙf›mòÕ¯~5»ï¾{úõë—-Z|îç|饗2jÔ¨<öØcyôÑG³téÒôïß?ƒ ÊG‘ 6Øàs?'P¿ „ð4a„Œ1"£FʳÏ>›•+WæK_úRºwïž®]»f›m¶IûöíÓºuë´mÛöS¿,dáÂ…™;wnfÏž9sæäÅ_̤I“2qâÄ,Z´(­ZµÊn»í–Ýwß=|p6Ùd“zü¤@]3ÀÜ{ï½—§žz*&LÈ /¼I“&åå—_ÎÊ•+?|OUUUš5k– ¤AƒY½zuV­Z•eË–}äg5kÖ,Ûm·]ºwïžnݺ¥W¯^éÖ­[ªªªêûcõÄ@k¡R©”ùóçgÞ¼y™;wnÞ{ï½,^¼8O?ýtn¸á†\{íµiÔ¨Q6Ø`ƒ4oÞ¹ãŽ;ŠÎʈ+ ‚¹‚*˜*˜*˜*˜*˜*˜*Ș1c2pàÀ¢3€2b € 2kÖ¬ÜyçEgeÄ@¬ªT*•ŠŽŠá B¨`B¨`B¨`B¨`B¨`B¨`B¨`B¨ ÇOUUUÑ@1@3@iß¾}?üð¢3€2RU*•JEGÅp!T0!T0!T0!T¨%K–”!T˜Ë/¿<{ì±GZµjUta‰L IDAT P|‹1T˜Õ«W§}ûöyóÍ7S×xýõ׳Å[Ôé9€ÿŒ+ ‚Ìš5+÷ÜsO6ÜpÃ:?ל9srì±ÇÖùy€ÿŒ*Ș1c2pàÀ:?Ï‚ ²ï¾ûæ­·Þªósÿ!T°©S§¦GiÔ¨Qºté’qãÆ}xì­·ÞÊgœ‘³Î:+çž{nz÷î“N:)o¾ùæ‡ïyá…²Ûn»å¢‹.Ê·¿ýí4lØ0K—.Í°aÃ2iÒ¤üùÏÎÉ'Ÿ\ÄG>#Ï € 2kÖ¬<ûì³ùÞ÷¾——^z)ßýîwsÆgdÚ´iéß¿vÜqÇŒ7. ,H=râ‰'æ[ßúV’dñâÅÙyç³dÉ’<÷ÜsiÓ¦M¶Új«¬^½:o¼ñF’äÄOÌ…^˜7Þ8UUUéرc¦OŸ^äGþ !T m·Ý6/½ôRjkkÓ Á_n,Úb‹-2gÎœ¬^½:ßøÆ7r饗æí·ßNË–-?üs·ß~{ ”SO=5W_}uš7ožE‹eØ°a9餓òÒK/e³Í6ˆnh €/·@ûë8˜$555©­­M’Œ=:IÒ´iÓ¼¿ÿþIþò,Ã$¹üòËÓ°aÃœzê©éÑ£G.\X/_€|~ „À?X³fM’äõ×_ÿÈë-Z´H’¬·ÞzI’ãŽ;.Ï?ÿ|öØcŒ?>»ì²K.»ì²zmþ3Bàì±ÇI’‘#G~äõ9sæ$IöÛo¿$ÉÅ_œvØ!øÃr÷Ýw§ªª*ßûÞ÷>|¿'@ùó B¨@[o½u^y啬^½: 6L’tèÐ!¯¾újjkk³páÂôìÙ3«W¯ÎĉÓ¬Y³$Éyç—G}4O>ùdÖ_ýl²É&™>}zš7ož$iß¾}Zµj• &d£6ʪU«2uêÔ´iÓ¦°Ï |º†\pÁEGõcÍš5¹îºërë­·fÍš5iܸq¾üå/禛nÊ-·Ü’R©”šššôë×/ÇsLæΛK.¹$/½ôRî¿ÿþTWWçúë¯O“&M’$ßüæ7sï½÷fùòåyðÁS*•rà 7¤yóæiÚ´iFŽ™åË—gï½÷.ø“ŸÄ„PÁ<ƒ*Ș1c2pàÀ¢3€2b € 2kÖ¬ÜyçEgeÄ@Ì3 ‚¹‚*˜*˜*˜*˜*˜*˜*˜*ÈðáÃSUUUtPF „PÁ „PAÚ·oŸÃ?¼è  ŒT•J¥RÑ@1\AÌ@¬QÑ@ÝX½zu–.]ú™Þ[UU•fÍšÕqPŽ „°–Z´hQ6Úh£ÏôÞ½öÚ+<òHåÈ-Æ°–jÕªUºuëö™Þ;hР:®Ê•ÖbçœsN4øôÿÛ_]]C9¤žŠ€rc €µØA”êêêO<Þ¨Q£ì½÷Þž?Ì@k± 6Ø ûî»o5úøÇ×ÖÖæ˜cŽ©ç* œ`-wôÑG§¶¶öcÕÔÔd¿ýö«ç" œ`-·Ï>ûdýõ×ÿ‡×«««s衇¦¦¦¦€* \`-·Î:ëdàÀÿð,ÂU«VeðàÁUåÂ@`РAYµjÕG^kÞ¼yöÜsÏ‚Š€ra € °ûeË–þ{ãÆsÔQG}â——•Ã@ aÆ9úè£Ó¸qã$ÉÊ•+3hР‚«€rPU*•JEGuï¹çžKÏž=“$mÚ´Éœ9sRUUUpP4W@…øÊW¾’¶mÛ&IŽ=öXã $ñÀX ­Y³&óçÏÏœ9s2oÞ¼¼÷Þ{Yºti6ÞxãÌ™3'7ίýë4mÚ4n¸aÚ¶m›¶mÛ¦iÓ¦E§õÌ-Æð·lÙ²Œ3&&LÈ /¼I“&åÕW_ýÈ·7lØ0n¸aªªª²téÒ4iÒ$µµµY²dÉG~ÖlÎ;gûí·O·nÝÒ³gÏì°ÃiÐÀÍG°¶2ÀÐóÏ?Ÿx üãóì³ÏfÍš5ÙrË-Ó½{÷tíÚ5[o½uÚµk—Í6Û,mÚ´É:ë¬ó៽ÿþû³ÿþûøï‹/Îœ9s2wîÜÌž=;S¦LɤI“ò /äí·ßN‹-Ò¿ÿì±Ç9øàƒÓºuë">2PG „ð1uêÔÜ|ó͹ýöÛóÚk¯eÛm·ÍW¿úÕì±Çéׯ_Üüꫯæüc{ì±<òÈ#Y´hQúöí›樣ŽrK2¬ „PÆÖ¬Y“‡z(W^yeþð‡?d«­¶Ê‘G™Ã?<]»v­×–U«VåÑGÍí·ßž{ï½7kÖ¬ÉqÇ—3Î8#;v¬×àóc €25bĈ|ç;ßÉÔ©S3`À€ :4 (‹o^²dIn¸á†üüç?Ï«¯¾šAƒåûßÿ~:tèPtð/ò¤a(3&LHïÞ½sÐA¥cÇŽ™¹l®üW¼ûî»9ûì³sã7fÈ!¹úê«SSSStð)\A»óÎ;Ó£G4kÖ,/¾øbN9å”/ä8˜$-Z´È 7Üûï¿?÷Ýw_z÷î×^{­è,àS @W\qEŽ8∠2$=öXÚµkWtÒçb¿ýöËóÏ?ŸªªªôîÝ;“'O.: øB(È~ô£œ}öÙ6lX.¿üòTWWô¹úÒ—¾”1cÆd‡vHÿþý3nܸ¢“€á„P€k®¹&C‡Í 7ÜÁƒS§V®\™ÁƒgôèÑ;vl:tèPtð7 „PÏî¿ÿþ|ðÁùÕ¯~•N8¡èœzQ[[›8 3fÌÈرc³ÑFü/!Ô£yóæ¥K—.9餓òãÿ¸èœzµxñâìºë®Ùj«­rÏ=÷ü/!Ô“R©”}÷Ý7«W¯ÎÈ‘#¿°ßTüŸxíµ×Ò­[·üüç?ÏqÇWt_Rõæ–[nÉرcsÝu×Uä8˜üå‹K.½ôÒœuÖYY°`AÑ9@\AõbåÊ•éرcÎ<óÌüŸÿóŠÎ)T©TJïÞ½Ó»wï\rÉ%Eç@Ås!Ôƒ_ýêW©­­ÍI'TtJ᪪ªòÃþ0×\sMæÎ[tT<!Ô±R©”K.¹$gœqFjjjêí¼‹-J«V­>ñ AþÙñº´çž{¦cÇŽ¹úê«ëýÜÀG Ž=õÔSyã7rä‘GÖëy5j”wÞy'K–,ù·Ž×µcŽ9&·ÞzkÖ¬YSÈù€¿0@ûío›]wÝ5mÛ¶­×ó6iÒ$mÚ´É6Ûlóo¯kGydfÍš•±cÇr~à/ „PÇÆŒ“]vÙ¥swíÚ5]»vý·×¥Ö­[§C‡yâ‰' 9?ðB¨CË–-Ë‹/¾˜^½zrþ³Î:+Mš4ù·×µ=zdܸq…0@zõÕWS[[[Øm¼ øŽ×µŽ;fÚ´i…6@¥3@z÷Ýw“$-Z´(¸¤<µhÑâÃßP !Ô¡E‹%Iš7o^pIyjÞ¼¹ f €:´þúë'I–/_^pIyzï½÷ }"` €:õ×+ßyç‚KÊÓÛo¿–-[Í@u¨mÛ¶I’yóæ\RžfÏžýáï(†êPëÖ­Ó¶mÛ<÷ÜsE§”¥ñãÇç+_ùJÑPÑ „PÇvÞyç<ýôÓEg”+VdâĉéÕ«WÑ)PÑ „PÇ:è <òÈ#yï½÷ŠN)+<ð@ª««óÕ¯~µè¨hB¨ctPJ¥RFŒQtJY¹ùæ›sðÁgƒ 6(:*ZU©T*k»ÓN;-/¾øbF]tJYxýõ׳Í6ÛdÔ¨QéÓ§OÑ9PÑ „PæÍ›—:ä¾ûîË^{íUtNá¾þõ¯gÞ¼yyøᇋN€Šg €zrÞyçå‘GɳÏ>›ÆS˜ &¤W¯^;vlvÚi§¢s â ž,]º4]ºtɱÇ›þð‡EçbÅŠÙi§²ë®»æšk®):ˆêÕÿøÇì½÷ÞùýïŸÝwß½èœzwòÉ'çøC&Nœ˜&MšÄ·@½Úc=rÖYgåCÉäÉ“‹Î©W_|q®¿þú >Ü8eÄ„PÏjkksÈ!‡ä…^Èc=–/}éKE'Õ¹[n¹%'œpBn¸á† <¸èào¸‚êYÆ sÛm·¥mÛ¶éÓ§O¦M›VtR6lXŽ=öØüøÇ?6@2@Ö[o½Œ92;wή»îšÇ{¬è¤ÏÝš5kòÝï~7§Ÿ~z®¸âŠœsÎ9E'Ã@YýõóÀdÀ€Ùk¯½òÓŸþ4kË€Þyçì³Ï>¹ä’KrÓM7åŒ3Î(: øžAeàÊ+¯Ì9眓Ýwß=×\sÍú¹„#FŒÈi§–êêêÜu×]Ùa‡ŠN>…+  :4cƌɜ9sÒµk×\vÙeY¹reÑYÿ’¹sçfàÀ9ðÀ³Ûn»eܸqÆAø0@™èÑ£G&L˜sÏ=7çŸ~¶Ûn»Üzë­Y³fMÑiŸêwÞÉ7¿ùÍl½õÖ7n\~ÿûß禛nJ‹-ŠN>·@š5kVþçþ'7ß|s:uꔡC‡fðàÁYo½õŠNûÐk¯½–aÆåW¿úU7nœï|ç;9å”SÒ¸qã¢Ó€ÊØ´iÓrÉ%—äÖ[oÍzë­—N8!GuT¾üå/ÒóÁ䡇ÊM7Ý”ûï¿?›m¶YN=õÔœvÚiiÒ¤I!MÀÆ@_o¿ýv~ýë_çÆoÌôéÓ³õÖ[gàÀÙk¯½Ò«W¯¬³Î:uvî äñÇÏ<ûî»/Ë–-Ëž{î™ÿþïÿÎA”† ÖÙ¹€ºg €/˜‰'æöÛoÏý÷ߟ)S¦d½õÖK¯^½²ÓN;eûí·O·nݲõÖ[ÿ[£á¢E‹òâ‹/fÒ¤I™8qbž}öÙLž<9묳NvÙe—zè¡9ì°Ã²ÑFÕÁ'Š` €/°ùóçgÔ¨QyüñÇ3a„¼øâ‹Y±bE’d“M6I›6mÒ¶mÛÔÔÔ¤yóæiØ°aš4i’Å‹gåÊ•Y¾|yÞ}÷ÝÌ›7/³fÍÊòåË“$m´Q¶ß~ûôìÙ3»ï¾{z÷îššš"?*PG „°Y½zu^~ùå̘1#³fÍʼyó2wîܼÿþûY´hQæΛiÓ¦eÏ=÷Luuuš4i’æÍ›g³Í6K»víÒ¶mÛtîÜ9mÚ´)ú£õÄ@døðá¹ãŽ;ŠÎÊHU©T*ÄPÁ „PÁ „PÁ „PÁ „PÁ „PÁ „PAÆŒ“”!TY³fåÎ;ï,:(#B¨`U¥R©TtP W@3@3@3@3@3@3@3@>|xªªªŠÎʈ*Ð /¼ÝvÛ-]tQ¾ýío§aÆYºti’déÒ¥ùÁ~ÿú¯ÿJß¾}Ó»wï<÷ÜsþÙeË–å›ßüfœo}ë[2dH.¼ðÂtêÔ)¥R)>ø`Î8ãŒl¾ùæ™5kVöÚk¯4jÔ(]»vÍøñã“$¿ýíoSSSóáX¹téÒüú׿þÈkË—/Ïï~÷»œp éÓ§On¹å–4oÞ<[n¹ež}öÙŒ=:½zõJuuu:w'ÖóoÖU¥R©TtP?ÆŒ“+¯¼2ãÇÏêÕ«óÆo$IN<ñÄ\xá…iÕªUößÿ\wÝuÙtÓM“$ƒ Ê#<’™3gfuÖIß¾}Ó¥K—üæ7¿IUUUV¬X‘M6Ù$K–,Éš5kòî»ïf›m¶É»ï¾›‹.º(C† ÉK/½”ÝvÛ-Ý»wÏ„ ’$;vÌŒ3ò·%ùÛ×Ö¬Y“·Þz+­[·NóæÍs÷ÝwgÛm·Mûöí³ñÆçÜsÏÍÉ'ŸœY³fe»í¶KïÞ½3zôèúÿ¥Àœ*PóæͳhÑ¢ 6,'tR^zé¥l¶Ùfyꩧ²Ï>û|쟹뮻òÊ+¯ä¼óÎËÔ©SÓ©S§ýýØ÷qãßV[m•×^{-kÖ¬I’l»í¶y饗>òž¿­T*¥Aƒéرc¦OŸž$éСC^}õÕøÙo¾ùfÞ{ï½Ïé7•Ã-ÆP.¿üò4lØ0§žzjzôè‘…ÿ{wWUøü}Ye\iqqr DÌRÓÔDr#3M{¸ŒãLM3mÓô5œ~Sf£6.“K:"¦eËd.¹àf¡â–+²¸Š"—ûû£‰ùúMKË{Ï•óz><pî9çó>÷/}?>ŸÏ)(¯¯¯vìØ¡6mÚÈf³ýàgàÀZ³f¤ïJºÿíÿîkx³}ÝÝÝu§ónv77·›Þ»´´ôŽî ¾CA˜Pbb¢ÒÓÓÕ£G}õÕWêÒ¥‹Þzë-Y­V}ûí·ºvíÚ®±Z­ºté’$©¨¨ÈÑ‘€P&4}útµmÛVëׯתU«d±Xôâ‹/*<<\—/_ÖìÙ³o8?//O³gÏV“&M$IŸ|òÉ Ÿ¿løç°Z­•¿———ÿìû€Ÿç‡sóTyo¾ù¦ÆŽ«ZµjiÀ€ ’¿¿¿ú÷ï¯àà`ýîw¿SNNŽ¢££uêÔ)­Y³F)))jÑ¢…V¯^­©S§ÊÛÛ[-Z´ÐæÍ›uöìÙîÿ}ah³Ù*— _þ}¬qãÆ:|ø°fÏž­G}TŸþ¹ $I{öìQëÖ­+¯ýßK“¿¿·Õj•««ë-Ç·‡„€ ;wN:uÒôéÓõüóÏ«eË–Z¹r¥¼¼¼´nÝ:ÅÅÅiÖ¬Y1b„ÒÒÒ´dÉÕ¬YSqqqZ¼x±|||” Ñ£G+<<\ 4¨¼÷²eË*ߎŽ?®ÐÐPÝêŸý‹EÊÈÈpp2à,˜ATa!!!jÛ¶­,ËM?wuuUbb¢ƒSgBATq#GŽ”««ëM?³Z­JHHpp"àLXb TqgÎœQÆ UQQqÃquêÔI©©©%΀„@W¿~}EGGÿ`¡ÅbÑÈ‘# Jœ!`ÇÿÁ1‹Å¢Ç{Ì€4À™P&0pà@¹¸ü÷Ÿÿ®®®êÕ«—j×®m`*à (ðóóÓÃ?,777I’ÍfÓСC Nœ!`#FŒÕj•$U«VMýû÷78p„€IôîÝ[žžž’¤G}T^^^'΀‚0ëׯëÚµkjݺµ$©ÿþ*,,48p›Íf3:€_¦¢¢BÇŽSFF†öîÝ«#GŽ(''GÙÙÙÊÍÍUYYÙ-¯õõõUPP6l¨F),,L­[·VëÖ­àÀ§F  îA6›M™™™Ú°aƒ6lØ Í›7«¨¨HîîîjÖ¬™š6mZYúÊËËK>>>rqqÑÒ¥Kõøã«¢¢B—.]Ò¥K—*ËÄÓ§OëÀ:}ú´$)$$D=zôP÷îÝ«zõêüäàn£ î!{öìQrr²V¬X¡“'O*00P±±±êÞ½»Ú¶m«ððpU«VíGïqíÚ5yxxüè9ùùùÚ»w¯vîÜ©7jÛ¶mºzõªºvíª!C†hРA”…T„€“»r劖.]ª™3gjÿþýjÞ¼¹âããõØc)""Â!®]»¦/¿üRÉÉÉúðÃU\\¬þýûkÒ¤IŠŽŽvH`„€“*..Ößþö7½óÎ;ºr劆 ¦gžyFíÚµ34×µk×´fͽóÎ;Úºu«Z·n­W^yEýû÷—Åb14¸s„€“)++Ó¬Y³4mÚ4•——kêÔ©7nœüýýŽö{öìÑôéÓ•’’¢ÈÈH%%%)&&ÆèXà¸À¥¥¥©C‡úÓŸþ¤Ñ£Gëøñãzá…œ²”¤víÚ)99Y»wï–ŸŸŸbccõä“O*??ßèhà6QN ¼¼\øÃÔ¹sghÿþýJJJR­ZµŒŽv[Úµk§Ï?ÿ\«V­ÒÚµk¦O?ýÔèXà6P;sæŒzôè¡Y³fiîܹZ¿~½BBBŒŽõ³<úè£ÊÌÌÔC=¤~ýúéå—_VEE…ѱÀ`BÀ@ûöíS¯^½T³fM­\¹RáááFGºk,X  &èÁTrr²<==Žn‚‚0HZZš~øaEFF*99Y¾¾¾FGºë¾úê+õéÓGáááúè£äíímt$ðPØ·oŸºu릞={jéÒ¥ªV­šÑ‘ìæðáÃzðÁªÿûßUúY¸Q–­N:©k×®zÿý÷åêêjt$»;yò¤ºuë¦èèh-Y²D‹ÅèHà?(*//W§Näêêª7ªFFGr˜ôôtÅÄÄè•W^Ñïÿ{£ã€ÿà-Æ€M›6MyyyúøãMUJRÇŽµxñb½øâ‹Ú¿¿ÑqÀ0ƒpŒŒ EFFê£>ÒC=dtÃ<ñÄÚ·oŸÒÒÒL±¼gGA8H¯^½Ô A-\¸Ðè(†ºté’š5k¦×_]£F2:¦GA8Àæ͛իW/9rDÁÁÁFÇ1ÜŒ3ôöÛoëðáüÕƒ±!௾úªFår°°°PþþþZ½zµ]¯ù%Ƨ+W®hñâÅÜ!`g§NÒ—_~©‘#G:d<777]¼xQEEEv½æ—ðôôTBB‚é—[à (;[¶l™7n¬N:9dÒ¶mÛÔµkW£ã€;Ä„€“»zõªþõ¯éwÞÑ×_­ÐÐP 2DƒVëÖ­–aÆ JNNÖG}¤ÒÒR 0@“'OVçÎ’Ø!pÙ»w¯’““•œœ¬cÇŽ©^½zŠULLŒÚµk§–-[ÊÓÓósþüyeddh×®]Ú¸q£¶oß®²²2=ðÀJHHРAƒäïïž‚¸GeffjãÆúòË/µiÓ&ÈÍÍMMš4QÓ¦M¬ÀÀ@5lØPÕ«W—ŸŸŸÜÝÝååå¥ÂÂB]¿~]%%%*((PNNŽNŸ>­ììleff*77W’Ô´iSÅÆƪ{÷îêÞ½» ~jp·QU€ÍfÓ‰'”‘‘¡ŒŒ }ûí·:}ú´rrr”››««W¯ÞòZ???5lØP5R£F¦Ö­[«uëÖª]»¶Ÿ‚0ŠŠ ]ºtIË—/×3Ï<£üü|¹ººÊ×××èhÀ`nF`...ªU«–jÖ¬)IªU«–Á‰€³`!`b.F` BÀÄ(£ LŒ‚01 BÀÄ(£ LdÙ²e²X,FÇN„‚01 BÀD‚ƒƒ5xð`£c'b±Ùl6£C03£ LŒ‚01 BÀÄ(£ LŒ‚0‘¬¬,¥¤¤8 BÀDRSSot àD(£ L¤k×®Z±b…Ñ1€±Øl6›Ñ!ƒ„€‰Q&FA˜!€;RTTdtpQ¸-3fÌP=äïïotpñc·tòäIýêW¿’$•——+88Xyyyâ¿TÌ L$55Uñññ·unvv¶FŽYù·›››|}}í ÄÍè'++K)))?yÞùóçÕ§O]»vÍ©€‘(:wîœþò—¿ÈÍÍMnnnÚ¶m›"""ôÊ+¯¨Aƒš={¶öîÝ«š5kjܸqz÷Ýwo¸þÀzâ‰'´gÏ5oÞ\ .T‡$IúÍo~£¸¸8]¾|YÓ§OWaa¡|||T\\¬·ÞzKYYY:räˆÊËË5cÆ EFFJ’JJJôꫯ*''GÁÁÁ:{ö¬BBB´téR«I“&IúnB///IÒk¯½&«Õª¤¤$UTThþüù•Ë’“’’TRRrñéÓ§ëÚµk:wîœ:uê¤éÓ§ëùçŸWË–-µråJyyyiݺuŠ‹‹Ó¬Y³4bÄ¥¥¥iÉ’%ªY³¦âââ´xñbùøø(!!A£GVxx¸4h`Ä×€i0ƒ€SûþÍÉü×û`!`"Ë–-ã­Àà„œZYY™¤Ÿ÷ÂðÓ( ÖàÁƒŽq[Ž=ª¤¤$8qB’4mÚ4]¾|ÙàTT=ìA˜3£ LŒ‚01 BÀÄ(£ LŒ‚0‘¬¬,¥¤¤8 BÀDRSSot àD(£ L¤k×®Z±b…Ñ1€±Øl6›Ñ!ƒ„€‰Q&æftöqñâE}ðÁ·u®¯¯¯ìœ8#ö ª¨ëׯ«Zµj’$ww÷=oôèÑš?¾£¢'Âc Šrwwט1cäîî®ëׯßòG’† bpZ`fUØæÍ›ó£çÔ©SGgÏž•«««cB§Â B  ëÖ­›êÕ«wËÏ«U«¦áÇS`b„@æââ¢aÆUîEø•••ièСNœ KŒ€*¾R‡núYPPNŸ>íàDÀ™0ƒ¨âÚ·o¯_ÿú×?8îîî®ÄÄDgBA˜@bb¢ÜÝÝo8výúu––fpøða5oÞ¼òo‹Å¢°°0íß¿ßÀTÀ0ƒ0fÍš©U«V²X,’$7771ÂàTÀP&‘˜˜(WWWIRyy¹ NœKŒ“ÈÉÉQ£Fd³Ù¥;w 8f&Ñ°aCµnÝZ’x{1¨Ä B  ÉÉÉÑ¡C‡”­ÜÜ\åååéܹs*--Õ•+Wôí·ßêÔ©SŠŽŽ–»»»üüüT½zuÕ¯__ 6Týúõ¢æÍ›ËÇÇÇèÇ@AܣΞ=«­[·jË–-úú믕™™©‚‚IRõêÕ¨ ¨^½zòðð···¬V«6oÞ¬ØØXIRAA®^½ª¼¼<åææêìÙ³²Z­²X,ºï¾ûÔ²eKEFF*::Z;vTõêÕ|d`„À=Âjµ*55U~ø¡>ÿüs:tHêر£Ú¶m«–-[*<<\-Z´PíÚµoyŸ“'OêW¿úÕM?«¨¨PVV–8 ÌÌLíß¿_iii•cEEEé‘GÑ€Ô¸qc;=)p$ BÀÉ¥§§kþüùZµj•.\¸ víÚ©_¿~Š‰‰Qdd¤Cfõ}?[qݺuZ³fÎœ9£V­ZéñÇרQ£T¯^=»göAA8¡«W¯jÑ¢EúÇ?þ¡¯¿þZmÚ´Qbb¢  ûî»ÏÐlÚ¹s§V®\©Å‹«¨¨Hýû÷ׄ mh6pç('RZZª¹sçê7ÞP~~¾†ª±cÇ*22Òèh7uíÚ5}ðÁz÷ÝwµuëVEGG륗^ªÜã8?£øβeËÔ¸qcýñT||¼Ž;¦ùóç;m9(I:t¨¶lÙ¢­[·ÊÃÃC=zôP=tàÀ£ã€Û@AìСCêÞ½»FŒ¡¾}ûêĉzóÍ7Õ A££Ý‘®]»jíÚµÚ¶m› Õ¶m[ýþ÷¿×•+WŒŽ~!` wß}WíÛ·Wqq±vìØ¡¹sçªnݺFÇúE:wî¬ôôt͘1C ,P‡”‘‘at,p „€Š‹‹5pà@M˜0AS§NÕÎ;z)ñrqqÑøñãµoß>Õ¯__QQQš3gŽÑ±ÀM¸0›¼¼<õéÓGçϟצM›ÔµkW£#ÙMÆ µ~ýz½þúëš8q¢Ž;¦7ÞxC‹Åèhà?x‹1à@GU\\œ|}}õÙgŸ)((ÈèH³jÕ* >\ýû÷×ûï¿/WWW£#Q“““£˜˜jÍš5ªY³¦Ñ‘nûöíêÝ»· ¤ùóç3“'@A8@qq±ºté¢5jhݺuòññ1:’avîÜ©=zhòäÉš6mšÑq0=^R8ÀøñãUVV¦?þØÔå $ÝÿýZ±b…Þxã }üñÇFÇÀô˜AØÙ¢E‹4~üx¥§§+<<Üè8NãþçôÖ[o)##C 64:¦EAØQ~~¾š6mª×^{MãÆ3:ŽS±Z­zðÁåïï¯ääd£ã`Z„€M™2EiiiJMMå…7qôèQµlÙR›6mÒý÷ßotL‰‚°“ãÇ«E‹Ú´i“:uêt×î[XX¨ÐÐPÍ›7O ¸£k­V«Þ}÷]-Z´Hááá:s挢££%I¾¾¾Ú¶m›ÊÊÊ”’’r×Æü)“'OÖîÝ»µmÛ¶»z_p{ÜŒTU³gÏV›6mîj9(Innnºxñ¢ŠŠŠîè:«Õª!C†hõêÕÚ°aƒbbbtùòeÍŸ?_“'OV^^ž^|ñE¥§§ßµ1oÇäÉ“ªÝ»w«C‡wýþàÇñcÀÊÊÊ´dÉ=ú®ßÛÛÛ[jÚ´é]7oÞ<}ðÁ9r¤bbb$}73Ð××W’ôÒK/iܸq ¹kcÞŽÅÆÆjÁ‚wýÞà§Qv°iÓ&]¸pAƒ¶Ëý#""qG×¼ÿþû’tCi¹nÝ:EFF*33Sk×®ÕóÏ?WǼ]ƒÖŠ+TQQa—û€[£ ì 55UaaaªU«–]î?eÊy{{K’Ö¬Y#}úé§?zÍåË—%I;v”$•––jãÆ Ó³Ï>«iÓ¦UÞó§Æ¼Ûºvíªüü|>^ß|ó¼½½Õ¶m[ýîw¿SII‰Ñ±îŠýû÷+66VcÆŒÑøñãµmÛ6…„„ Ü!` ûî»O[·nÕ;ï¼£÷Þ{OM›6Õ¼yóTVVft´Ÿ%''G&LPÛ¶muùòeíرCýë_åîînt4p „€Á\\\4nÜ8>|XÔ„ Ô¤IÍ™3GW¯^5:Þm9uê”&L˜ Æë£>Òœ9s´sçNEFF ü‹Íf³Àegg+))I ,P54räH3FaaaFG»ÕjÕgŸ}¦¹sçêßÿþ·‚‚‚ôÜsÏiôèѪV­šÑñÀm¢ œÔ¹sç´hÑ"Í›7Oß~û­"##5pà@ 8PMš41$“ÕjÕæÍ›µzõj­ZµJgÎœQÏž=5vìX=òÈ#,%àDA89›Í¦Í›7kåÊ•Z½zµrssÕ¢E ÅÄĨk×®ŠŽŽVÆ í2¶ÕjÕÞ½{µuëVmÙ²E›6mR~~¾:tè jÈ!úõ¯m—±€cP÷›Í¦]»véóÏ?×–-[´sçN•––Êßß_­ZµRXX˜Z´h¡F©Aƒ T½zõäêêzË{–––*77WyyyÊÍÍÕÉ“'µÿ~effêàÁƒ*--Uƒ Ô­[7EGG«oß¾ vàS{¢ îaׯ_Wzzº¾ùæíÛ·O™™™:pà€.^¼xÃynnnòññ‘‹‹‹\]]uýúuUTTèÒ¥K7œçêêªàà`…‡‡+<<\ŠŒŒ4lI3°? B  *--U^^žòòòtöìY]½zU%%%Ú¾}»-Z¤üã²X,òóóSõêÕ+gÖ­[÷Gg€ª‡‚0‘eË–iØ°aâ¿à{.Fà8]»vÕŠ+ŒŽœ3c!`b„€‰Q&FA˜!`b„€‰Q&’ššªøøx£c'BA˜HVV–RRRŒŽœ!`b›Íf3:c0ƒ01 BÀÄ(£ LŒ‚01 BÀÄ(£ LdÙ²e²X,FÇN„‚01 BÀD‚ƒƒ5xð`£c'b±Ùl6£C03£ LŒ‚01 BÀÄ(£ LêäÉ“FGN€‚0‘¬¬,¥¤¤(;;[#GŽ4:pnFà8©©©6l˜Zµj¥k×®8f&´wï^9sFãÆ«<–™™©~ýú饗^ÒSO=¥:hÛ¶m•ŸWTTèõ×_דO>©I“&ÉËËK‹¥òÜ›,6›ÍftŽ‘••¥]»v)>>^Íš5Ó¡C‡*? –§§§Ž9"›Í¦   U¯^]G•$%%%éå—_VQQ‘<<<4þ|3FÇ×’%KŒz$ð ±Ä0‘àà`ßô³ &ÈÓÓS’d³Ùäéé©ãÇW~¾víZUTTÈÕÕUnÅè_ IDAT’4hÐ 3Fß|óýƒ»¡  IzöÙgUXX¨3fÈÅÅE×®]Óÿ^pÔ¥KmÚ´Ik×®UŸ>}T^^.IŠ‹‹3*2¸ Xb ˜ÅbùÁã7*!!A+V¬PLLŒš7o®Ã‡W–„åååJJJÒÛo¿­±cÇêøñãjÚ´©þô§?©ZµjF= ø…˜A˜Ôÿ+ðÄOÈÛÛ[1117ý¼¢¢B—.]ÒîÝ»uß}÷9*&°3 BÀ„üýýuöìYåææ*00P’TXX¨²²2effjß¾}ºxñ¢$éÛo¿•fÎœ©O?ýT­ZµR@@€|}}U³fM…††ÊÃÃÃÈÇ¿€‹Ñ8Þk¯½&«Õª¤¤¤ÊcÓ§O—»»»úõë'???Mœ8Qnnn5j”,‹¢¢¢tîÜ99R?ü°ºt颖-[* @ÿüç? |ðK°!€Ÿd³Ùô÷¿ÿ]6›M“&Mª|¸ªU«&I*++SBB‚Á©€3°Øl6›Ñ!Ø_ZZš¢¢¢$IÊÎΖÅb1803“èر£‚‚‚$I#GŽ¤’$6ª ŠŠ ={VÙÙÙÊÍÍÕ•+WT\\¬ºuë*;;[ÕªUÓüùóU³fMùúú*((HAAAªY³¦ÑÑ€ƒ±Ä¸Ç•””(55U{öìQFF†öîÝ«cÇŽÝðÖbWWWùúúÊb±¨¸¸XÞÞÞ²Z­***ºá^>>> WëÖ­ÕªU+EEE©mÛ¶rqañU!pJOO×'Ÿ|¢ 6h×®]ª¨¨PHHˆÚ´i£ˆˆ5iÒD5RÆ (Êk?þøc=òÈ#•_ºtIÙÙÙÊÉÉÑéÓ§•™™©½{÷*##C.\PíÚµ£=zhÀ€jР ì„‚¸G8p@K–,Qrr²Nœ8¡æÍ›ëÁT=m—åÁÇŽÓ† ôå—_ê‹/¾Paa¡xàÅÇÇkèС,I    œXEE…>ûì3Íœ9SëׯWãÆõøãkðàÁŠˆˆph–ëׯkݺuJNNÖ‡~¨ŠŠ %&&jâĉjÖ¬™C³€»‡‚pRk֬џþô'8p@½zõÒ¤I“Ô«W/§xûpQQ‘.\¨¿ÿýï:vì˜ôç?ÿY¡¡¡FGwˆ†'³gÏuîÜY>ú¨š5k¦}ûöé³Ï>ÓC=äå $ùúújÒ¤I:tè’““µgÏ………i„ ºté’ÑñÀ  œDii©þð‡?(**JnnnJKKÓÊ•+ft´[rqqÑc=¦}ûöiÎœ9JIIQxx¸>üðC££€ÛÄcÀ :tH=ö˜NŸ>­¤¤$7Îif Þ‰üü|ýö·¿Õ¢E‹ôä“OjÖ¬Yòôô4:øÌ  –’’¢ÈÈHùùùiÿþý?~ü=YJRíÚµµpáB}üñÇúè£Ô¹sg8qÂèXàGPzûí·5dÈ=ùä“úòË/Õ¨Q#£#Ý}ûöUzzº,‹:wî¬}ûö Ü!`iӦ鷿ý­fÏž­3fÈÝÝÝèHwÕ¯ýk¥¦¦ªmÛ¶Š‰‰ÑîÝ»Žn‚=Ì™3G“&MÒÂ… 5lØ0£ãØUYY™† ¦Í›7kûöí 5:ø_(ûøã5`ÀÍ›7O£F2:ŽCX­Võë×OGŽÑöíÛ`t$ð„€åææªeË–zúé§õúë¯Ç¡.]º¤nݺ©qãÆZ½zµÑqÀPb³ÙÔ§O•——kíÚµ÷웊‰'N¨U«Vúûßÿ®ÄÄD£ãñ’ÀaÞÿ}mß¾] ,0e9(}÷â’7ß|SS¦LÑùóçŽÄ BÀ!ÊÊÊÔ¬Y3ýæ7¿ÑäÉ“Žc(›Í¦Î;«sçÎúÛßþftL„€Ì›7OV«UO?ý´ÑQ g±Xô—¿üEsæÌQNNŽÑq0= BÀÎl6›þö·¿iâĉòôô´ûx………ò÷÷¿£üœk~‰¸¸85kÖL³fÍrÈxàÖ(;Û¶m›N:¥ÇÜ!ã¹¹¹éâÅ‹***²ë5¿Ôˆ#´téRUTT8lLðC„€-_¾\ݺuSPPCÆóööV`` š6mj×k~©Ç\YYYÚ¾}»ÃÆ?DAØYjjªºtéâÐ1#""a÷k~‰ (44T[¶lqؘà‡(;*))Ñþýûuÿý÷;tÜ)S¦ÈÛÛ[’´fÍùøøèÓO?½ík%22R»wïvè˜àF„€;vLV«Õ¡Kw%©W¯^•¿«¤¤DÅÅÅ·}£4kÖLtø¸à¿ÜŒTeùùù’¤Úµk–aØ°a0`€jÔ¨aX†[©]»våwŒÁ BÀŽ %IµjÕ24‡3–ƒÒwß !Æ¢ ìÈËËK’tùòeƒ“8§+W®8|ßCp# BÀŽ¾Ÿ9xñâECs\¹rÅÐñoåÂ… ªS§ŽÑ105 BÀŽ‚‚‚$I¹¹¹†eX¾|¹¼½½•’’bX†[9}útåwŒAAØQƒ ¤´´4Ã2Ô¨QC^^^òðð0,í|õÕWêر£Ñ105‹Íf³¨Êâããe³ÙœrŸ‘®^½ªš5kjÙ²e4hÑq0-fvöè£ê‹/¾pÚ}òÉ'ŸÈÝÝ]>ø ÑQ€ÿÏÞ}‡EyæÝ?TQ@Aš…bE£±@ÄF”Þ˜MÁ4ͦo’-IÞd£yÝÍ&Ù”I\uÕD33‘*° h ‚ "R•&u`˜ßù1¯Ä®ÀC9Ÿëš œ™c"<3g¾Ï} j,‰ˆˆˆˆzX`` T*bcc…ŽÒ§üôÓO °aÄŽBDD4¨ñc""""¢^ðÊ+¯ 77iiiBG銊Šàè舔”Ì›7Oè8DDDƒ'‰ˆˆˆˆzAhh(233±ÿ~¡£ô ëÖ­ƒ»»;ËA""¢>€„DDDDD= ¥¥éé鈋‹Cbb".]º„éÓ§Ž?]]] çÔ©S˜;w.Ž=ŠÇ\è8DDDƒ'‰ˆˆˆˆºIii)6oÞŒÀÀ@˜™™ÁÃÃééé‰D8rä<ˆêêj¬[·N訂iiiÁ3Ï<ƒ… ¢ººJ¥RèHDDDƒž¶Ðˆˆˆˆˆú«ŽŽœ8q ˆGVVôõõ±hÑ"|öÙgðññM—ïùïÿ ///,Z´‹/(¹pÞ|óM´´´@OO077Gpp0$ \]]¡©É""¢ÞÆSŒ‰ˆˆˆˆ@]]öíÛ‡ÄÄD$$$ ªª vvvðöö†¯¯/-Z}}ý»ÞÇŸÿüglܸ‡ÂÔ©S{)¹ð>ýôS|øá‡8tèfÏž‹/B&“A&“!77ÖÖÖ‰D‰D˜5k–Ðq‰ˆˆ „DDDDD÷pþüyÄÇÇ#11‡BGG\\\àããŸ.ù”J%–-[†ììl> Ø·o0uêTx{{ÃÇÇ®®®ÐÒÒêÖÇlllDPPN:…ˆˆ,Z´¨[ï_høðñ~ýz|ýõ×xíµ×èûU*Ž9™L†ÈÈHTTTàñLJX,Fhhè-ë;ÑÃaAHDDDDƒÖ¹sç‡ØØX;v ºººX´h|}}áíí1cÆôx…BU«VA.—cýúõx÷Ýw¡¡¡ÑãÛÓ®_¿Ž•+W"-- ›6mÂSO=õH÷§T*‘’’¹\Ž]»v¡®®®®®H$X¾|9FÕMɉˆˆ„DDDD4h´··ãСCˆGLL .^¼333øúúÂ××:t¨ Ù¾ùæ¼óÎ;X¼x1¾ÿþû~½.all,^yåèèè ** =öX·Þ¿B¡ÀÞ½{!“É‹ææf,^¼b±AAA011éÖÇ#""èXÑ€VWW‡½{÷"&&III¨®®Æĉáïï???Ì;·ÛO~X™™™xî¹çPTT„uëÖá•W^®®®Ð±îÛÕ«WñÖ[o!""O?ý4¾úê«G^«ñ^š››™L†ÄÄD¨T*,]ºb±þþþ044ìÑÇ'""XÑ€STT¤>u8==˜7o|}}Ч7ºP(øûßÿŽõë×ÃÆÆü1$ 455…ŽvGׯ_ǧŸ~Šo¿ýøþûïáááÑë9êëë±{÷nÈårìß¿ÚÚÚðóóƒX,†——ôôôz=QÀ‚ˆˆˆˆú=•J…'N 66ñññÈÎÎÆðáÃáéé ???x{{÷ø$[w+..Æ;＃¨¨(Lš4 ¯¿þ:V®\ ¡£©â»ï¾Ã¦M› ­­çž{Ÿ|òIŸ˜z¼~ý:¢¢¢ “É––CCCB,ãÉ'Ÿ„ŽŽŽÐ‰ˆˆú „DDDDÔ/577#99qqqˆ‹‹CYYìììàçç,X° OU«¤¤‹-ÂÈ‘#1eÊìܹXµjV¬X3f’«µµ‰‰‰øñÇ+++üáÀÙ³g±gÏìß¿Ó§O$Û”••!""2™ ÇŽÈ# ‘H„ ôééL""¢ÞÀ‚ˆˆˆˆúÊÊJÄÅÅ!>>ûöíCss3fÎœ‰€€øúúö¹bêa•••aáÂ…ÐÑÑÁÁƒaff†k×®aóæÍؾ};òòòààà€ÐÐP,Y²sçÎÅ!Cz,OUURSSÕ›»444àÉ'ŸÄ‹/¾ˆÀÀ@hii¡©© ~~~8sæLŸ, ;A.—C&“!++ ––– …X,Æœ9sÄÒDDDŠ!õigÏžE\\bbb™™ ]]]¸»»Ãßß¾¾¾=z´Ð»UUU.\¥R‰ÔÔTXXXÜò5YYYË刋‹ÃÙ³ga``€¹sçâñÇÇ´iÓàìì ‡‡* kkk‘››‹3gÎ ++ ÇGNN† ‚'žxË—/Gpp0ÌÌÌnùÞþRv:þŸˆˆ¨;± $"""¢>¥££ÇŽÃîÝ» .ÀÜܾ¾¾ðõõ…‡‡GŸZ‡¯;UUUÁÝÝÍÍÍHMM…••Õ=¿§¢¢)))HMMÅ©S§››‹––À¨Q£0zôhX[[COO&&&ÐÒÒ‚¡¡!êêê P(ÐØ؈êêj”––¢¸¸333L›6 sæÌÁâÅ‹áêêz_›|ô·’°Sgé*“ÉPTT„ &@,C$a„ BÇ#""êQ,‰ˆˆˆHp …)))ˆŽŽFll,ÊËËaoo   ÀÅÅeÀ¯W]] wwwÔ××#-- ÖÖÖu?ííí(((@~~>Š‹‹QZZŠ«W¯¢¹¹µµµhkkCcc#Œ¡££CCC˜˜˜ÀÊÊ 666°¶¶ÆäÉ“i2³¿–„ÀoÞ?~2™ ááá(++ÃôéÓÕeá˜1c„ŽHDDÔíX‘ êëë‘””„èèh$&&âƘ1c‚‚‚ˆÉ“' ±×ÔÕÕÁÝÝUUUHOO‡Ð‘Y. ;utt ==2™ ‘‘‘¨®®Æܹs!‰ KKK¡#u „DDDDÔk***ƒÝ»w#%%J¥óçÏG`` `kk+tÄ^WWWOOO””” 55ãÇ:R·%a§¶¶68p2™ 111¸qã,X±XŒåË—cäÈ‘BG$""zh,‰ˆˆˆ¨G]¼xÑÑÑؽ{7222 §§‡¥K—"00¾¾¾ƒºXihh€§§' ‘šš ¡#u»TvjiiÁž={ “ɇööv<ùä“‹Å ÄðáÃ…ŽHDDô@XQ·;uêvïÞÝ»w#''#FŒ€¯¯/ô&#¢©© ^^^8þ§µµ8yò$’““1mÚ4¡#õšÁPvª¬¬Ddd$¤R)Ž= ###A,cñâÅÐÒÒ:"‘ B""""ºo555HHHÀîÝ»‘””„ææfÌž=[] ::: ±OS( ÂÑ£G‘œœŒ3f©× ¦’°Ó•+W¹\Ž'N`Ô¨Q†X,†««+455…ŽHDDƒ B""""º««W¯"&&ÑÑÑHKKƒ††.\ˆ   øûûcôèÑBGì BCC‘ššŠýû÷cÖ¬YBGÌ`, ;]¸pr¹R©gÏž… BCC!‹ñøã ˆˆ)„DDDDt‹ââbDFF"** ÇŽƒ¼¼¼ooo ±_Q*•‰DØ·oöîÝ ¡# n0—„Ξ= ©T ¹\Ž .ÀÁÁ¡¡¡H$ƒb]J""ê;XàÒ¥KˆŒŒDdd$Nž< ###`Ù²eXºt)ôôô„ŽØ/)•J¬\¹qqqHJJ‚›››Ð‘ú –„ÿçĉËåÇ•+W0eʈÅbˆÅbŒ?^èxDD4À± $"""ÄΟ?¨¨(DFFâôéÓ9r$ wwwîºúˆ”J%V­Z…¨¨($$$`áÂ…BGêsXvÕÑÑ£GB*•"22•••˜5kD"D"¬­­…ŽHDD B"""¢A&77»víBDDrssannŽÀÀ@„„„`áÂ…ÐÖÖ:†R©ðüóÏC&“!&&K–,:RŸÅ’ðö”J%RRR “ɺº:<ñÄ‹Å ™™™Ð‰ˆh€`AHDDD4dee!** QQQøõ×_1zôh!88nnnÐÒÒ:•R©°zõjlß¾111ðôô:RŸÇ’ðîZ[[±oß>H¥RÄÅÅ¡¹¹îîî‰DX¶l×%"¢G‚ˆˆˆh€:yò¤z£‘ .ÀÆÆË—/ÇòåËáêê MMM¡#H*• ¯¿þ:þóŸÿ 22~~~BGê7XÞŸææfÄÅÅA&“aÏž=P©Tððð€X,†ŸŸ …ŽHDDý B"""¢B¥RáøñãêR°¨¨cÆŒAHH–-[†9sæ@CCCè˜Þ[o½… 6@&“aÙ²eBÇéwX>˜ºº:ÄÄÄ@*•"99:::ðóóƒH$‚··7† "tD""êXõc8räˆúôá’’ØÛÛ#$$Ë—/ÇÌ™3…Ž8¨¼ûî»øòË/!•J"tœ~‹%áù~ý:"##!“ÉžžŽaÆ! ‰îîîÐÑÑ:"õQ,‰ˆˆˆú¥R‰´´4ìÚµ QQQ(//ÇĉŒàà`8;; qPúàƒðü?þø#V¬X!tœ~%á£)--EDDd2Ž?Ž‘#G"88"‘óçÏçDDÔ B"""¢~ ½½)))ˆŠŠBtt4ªªªàììŒåË—#88“&M:â öñÇcíڵزe „Ž3`°$ìEEEÉdÉdÈÎÎÆèÑ£‰D‚Ù³gsé""bAHDDDÔW) 8p‘‘‘ˆÅõë×1sæLõF#ŽŽŽBG$ëׯÇ|€M›6áùçŸ:΀Ò°{ååå©ËÂóçÏcìر‰D‹Å˜6mšÐñˆˆH ,‰ˆˆˆúööv$''#<<»víB]]fÍš…àà`,_¾ãÆ:"ÝäóÏ?ÇŸþô'|÷ÝwX³fÐq,–„=#++ R©ááá(**Âĉ!‰ ‘HøÑ Ã‚ˆˆˆH`HOOGxx8"##QUU…Ù³g#44!!!°µµ:"ÝÆ×_7ß|_}õÞxã ¡ã x, {ŽJ¥Â±cÇ “ɲ²2LŸ>‰"‘vvvBG$"¢Æ‚ˆˆˆH/Èår9"""PZZŠiÓ¦!44b±˜“‚}Üwß}‡W_}Ÿþ9þøÇ? gÐ`IØó”J%ÒÓÓ!“É…êêj¸¸¸@$!$$–––BG$"¢À‚ˆˆˆ¨:u r¹\}Jß„  ‰ ‰0qâD¡ãÑ}ؼy3^zé%¬_¿ùË_„Ž3è°$ì=mmmØ¿?d2bbbÐØ؈ùóçC"‘`ùòå1b„Љˆ¨›° $"""êagÏž…\.‡\.G~~¾zS‘HÄr£ŸÙºu+žþyüíoÇ~(tœA‹%aïkiiAbb"d2âããÑÞÞŽ%K–@,# Ç:"=„DDDD=   áááËåÈÉɵµ5BBB ‰0gΡãÑCرcÂÂÂðÞ{ïaíÚµBÇôX §¡¡111ËåØ»w/´´´àåå‰Dèëë ‘ˆˆ B"""¢nrùòeDDD@.—ãäÉ“5j–/_‘H„yóæASSSèˆôÂÃñbÅ ¼ýöÛøôÓO…ŽCÿKBáÕÔÔ`×®]Éd8xð ôõõ±XŒ¥K—BWWWèˆDDtX=‚²²2DDD <<G…‰‰ ‚‚‚ ‰°xñbhii ‘QTTÄb1^{í5üë_ÿ:ýK¾£¢¢‘‘‘Éd8räˆú÷¡X,Æ¢E‹øûˆ¨cAHDDDô€®]»†¨¨(Èår¤§§cèС@hh('f˜˜˜„††bÍš5øꫯ ¡¡!t$º –„}Ï•+WÔk¯vNT‡„„@,ÃÕÕA£M IDAT•?KDD} B"""¢ûPWW‡èèhÈår$''C[[~~~‰Dðòòz 5·”J%æÍ›‡ƒBOO¯Sӣسg†~øá–B£±±ß}÷är9:::`ll •J'''L˜0W¯^ÅgŸ}&PúÁ§§JÂÇcß¾}X·nàùçŸG@@üüüºåþƒ‚‚ÈårÈd2œ={666‰D‹Å˜9s¦ÐñºMFF¶oߎ7>øààñÇ8ѽ± $"""ºƒ††ÄÅÅA.—#)) àéé ‘H???>ÔýîÞ½AAAØ´i^xá…îŒLÝdß¾}ÀŠ+°yóæ[ÊÁ .ÀËË –––ؼy3*• ±±±xá…àïï-[¶ÐêÉI±cÇ¢¨¨---2dH·Ýï`“““£. /^¼ˆÅbˆD"Lž ‘H„ÀÀ@?òcøûû#++ †††ÈÍÍåæ%}LJJ üüü‚-[¶Ü²nZSS¦OŸmmmœ>>HLLDiiiý;¡G×Ý“„œ ìJ¥@xx8víÚ…úúzÌ›7b±ÁÁÁ033:â=q‚ˆú#NÑ RPP€?þNNN˜3g’’’ðòË/£  xýõ×{¬ô©ªª‚R©„­­- ñòË/#%%…N}@ff&¼½½±dÉìܹóŽå \¾|`nnÞ[ñè!p’°ÒÒÒ‚‡‡¶lÙ‚òòrDGGÃÊÊ ï¾û.,--áé鉭[·¢¶¶Vè¨DD B"""ð***ðÍ7ß`îܹpttÄÆáííÌÌLäååáÃ?„½½}çøÏþƒW_}Uýç×^{ C† Á_|ÑãMwöË/¿`éÒ¥˜7od2tuuïúõÚÚÚ€æææÞˆG€%aÿ6dÈøûûãçŸFee%vîÜ ¼üòË°°°@`` ¤R)…ŽJDÔïñc"""Ÿþû÷ïÇСC„+VÀÝÝý®b=A¡P`̘1(++»å6mmmÂÚÚºW3••wwwÌž=111÷, $$‘‘‘øå—_0cÆŒ^HIª;N7æ)Æ}G]]vïÞ ™L†äädèèèÀßß"‘^^^‚ï2ÍSŒ‰¨?â! mmmˆÇŠ+`nnŽ^xºººØ¹s'ÊË˱mÛ6,]º´×ËAˆˆˆÀÛo¿ •JÕå²cÇ´··ãßÿþw¯gìrss±téR̘1QQQ÷U€¯¯/ ..®'ãQ7â$áÀbdd„gŸ}{öìAii)¾øâ ”——cùòå5j””„ööv¡£ÞVUUÚÚÚ„ŽADÔ'‰ˆˆ¨_S©TÈÈÈÀÎ;!—ËQ]]yóæaÅŠ ň#„Ž•JÄÆÆÞ²n]kk+lll P(påÊ 6L ”ƒË¯¿þŠ… bâĉHLL¼ãnÄ·ÓÚÚŠ©S§âÚµk8sæÌm'?;::±XܱéuNfggãÀ4IÈ Â¾ïêÕ«ˆŒŒ„T*Eff&FŽ‰„††bþüùÐÔìùù•JMMÍ;NªT*¼ð شiS¯ä!"º_üDDDDýÒÅ‹±víZ888à‰'ž@zz:Þ~ûm!==kÖ¬éå ÄÇÇcøðá·ÝÔbÈ! E]]6nÜ(@ºÁ'??îîîpttDBB•ƒÀoÿÏbbb`hh777$%%¡££Ào/þOœ8•+Wò”ñ>¨s’pÚ´ixòÉ'h’°¥¥¥ËGê{¬¬¬ðÆoàرc¸xñ"þøÇ?âèÑ£X´hlmmñÖ[oáøñã=š¡smÒÛM/677ãý÷ßGkk+ËA"ês8AHDDDýF]]är9~úé'9r£F‚D"Á3Ï<óPkŠõ†˜˜¼üòËÐÒÒÂ'Ÿ|‚gŸ}¶Ëí øôÓOqèÐ!â³Ï>Ú5kJ;ð]¸p‹-‚ ’’’0|øð‡¾¯ºº:|õÕWˆŒŒDUU,,,`bbWWW¼ùæ›033ëÆäÔd’ðÈ‘#HJJÂ'Ÿ| òeËàçç×[qéýúë¯ËåJ¥ÈÏÏÇرc!‹!‰0mÚ´n{œŒŒ lÙ²[¶lÌœ9ÆÆÆ~[777ضmÛ-Ç""¡± $""¢>­­­ {÷îÅ?þˆØØXhjj"00O=õ<<<YOú§¢¢"ÌŸ?£FÂ`dd$t$УœnLý×éÓ§!“É —ËqùòeLš4 "‘b±ŽŽŽBÇ#" B"""ê“~ùåüøãÉd¨ªª‚›››zr‡Å=¨+W®`Á‚066Frr2LLL„ŽD}KÂÁ«sýZ¹\Žððp”——ã±ÇƒD"Ahh(ììì„ŽHDÔ«XQŸqåÊüüóÏøñÇqîÜ9899aåÊ•xöÙgakk+t<ê§JJJ°hÑ" 99¦¦¦BG¢>„%!)•J¤§§C*•"** 555puuEhh(BCCaaa!tD"¢Ç‚ˆˆˆÕÐЀ¨¨(ìر)))066ÆŠ+°bÅ ¸¸¸ú¹²²2,\¸:::8xð פÛbIHÚÚÚ°oß>ÈårÄÄÄ ±± ,€D"Á²eËúÌæWDDÝ!õºŽŽ¤¤¤`Û¶mˆ‰‰A[[¼¼¼OOO 2Dèˆ4TUUaáÂ…P*•HMMåÝKBú½ææfìÙ³R© hooÇÒ¥K!‰ˆaÆ ‘ˆ¨Û° $""¢^sñâElß¾Û·oGqq1\\\°råJH$NeP·ªªª‚»»;š››‘šš +++¡#Q?À’îäƈ…L&þ}û ¥¥oooˆÅbøøø@___èˆDD„!õ¨††DFFbëÖ­8tè,--ñôÓO#,, &L: @ÕÕÕpwwG}}=ÒÒÒ`mm-t$êGšššàïשּׁ,–„t[ÕÕÕˆŽŽ†T*Ejj* ‘Hèèè‘ˆè± $""¢n§R©pèÐ!lݺ‘‘‘hkkƒ¿¿?ÂÂÂàáá---¡#ÒUWWwww\»v iii܉” KBº_ˆˆˆ€L&ÃÑ£Gabb‚   H$,\¸Ç;"ê7XQ·)..VŸB|ñâEÌœ9aaaX±bO!¦WWW\½z©©©?~¼Ð‘¨cIHª¸¸áááÉdøå—_`aaˆD"¸ººBCCCèˆDDwÄ‚ˆˆˆIss3víÚ…mÛ¶!%%¦¦¦X¹r%V­Z…©S§ ‰††xzz¢°°©©©ppp: , éa@&“A&“áܹs°µµEhh($ f̘!t<"¢[° $""¢‡’‘‘mÛ¶A.—£©© ÞÞÞXµj¼½½¹þõª¦¦&xyyáüùóHMMåÚ–Ô­XÒ£ÊÉÉQ—…—.]‚££#D"Äb1&Mš$t<"",‰ˆˆè”––⧟~¶mÛ——‡©S§bÕªUX¹r%ÌÍÍ…ŽGƒP箳9998xð &Ož,t$€XRwÉÌÌ„L&CDDJJJàìì ‘H‰D‚±cÇ ˆ1„DDDtWJ¥{öìÁæÍ›‘˜˜ˆaÆaÅŠ ÃÌ™3…ŽGƒXkk+pòäI$''cÚ´iBG¢Œ%!u§ŽŽ>|2™ ‘‘‘¨ªªÂìÙ³!‹ +++¡#Ñ Ã‚ˆˆˆnëâŋغu+¶nÝŠ²²2,^¼«V­Bpp0† "t<ä ‚‚‚pôèQ$''sM/ê, ©'´··#99r¹ÑÑѨ¯¯‡››D"‚ƒƒaff&tD"X‘Zkk+¢¢¢°uëV$''ÃÒÒaaaxî¹ç¸#,õ …¡¡¡HMMÅþýû1kÖ,¡#Ñ Â’zRkk+’’’ “ɇÖÖV¸»»C,#((FFFBG$¢Š!!''›7oÆŽ;P__<ÿüóðöö†–––ÐñˆÔÚÛÛ!‹±ÿ~$%%ÁÅÅEèH4±$¤ÞÐÔÔ„ØØXÈårìÙ³àéé ±X ??? :Tà„D4° $""¤nܸ©TŠ-[¶ 33ãÇÇ‹/¾ˆ§Ÿ~£G:Ñ-”J%V®\‰¸¸8$%%ÁÍÍMèH4ˆ±$¤ÞTWW‡èèhÈår8pC† ŸŸÄb1<==¹ô=2„DDDƒLFFþóŸÿ 22J¥Ë–-à /¼€ @CCCèxD·¥T*±jÕ*DEE!!! .:KBĵk×¹\ŽC‡aØ°a ‚X,†»»;´µµ…ŽHDý B""¢A ººÛ·oǦM›ð믿bÚ´ixá…°råJ˜˜˜è®T*žþyÈd2ÄÄÄ`É’%BG"RcIHBºzõ*""" “Épüøq˜™™!88"‘nnnÐÔÔ:"õ,‰ˆˆ°#GŽ`ãƈˆˆ€®®.$ ^|ñEÌœ9SèhD]( èêêÞr½J¥ÂK/½„ü111ðôô Ñݱ$¤¾ °°R©áááÈÎΆ••BCC!‹1{öl¡ãQÇ‚ˆˆh€©««ÃŽ;ðÃ? 773fÌÀš5k ‘H`hh(t<¢[äååá駟ÆÞ½{1bÄõõ*• ¯¿þºú”x???SÝKBêKÎ;¹\™L†üü|Œ;‰"‘ÎÎÎBÇ#¢>ˆ!ÑqâÄ lܸ2™ ‹±zõjÌš5KàdDw†íÛ·còäÉ8xð ÌÌÌo½õ6lØ™L†eË– œ’èÞî§$üꫯ°zõjèëë £S§NA&“!<<—/_ƤI“ ‹!‰àèè(t<"ê#Xõc J¥øá‡pêÔ)L™2kÖ¬ÁSO=###¡ãÝSII ÆŽ‹öövèèè`ܸqHKKÃ矎/¿üR©!!!BÇ$ºow+ ?úè#¬]»ß|ó ^{í5SÒ`¤R©pôèQÈårDDD ¼¼3fÌ€D"Ahh(lmm…ŽHDbAHDDÔeggcãÆعs' BBB°zõj<ñÄBG#z o¿ý6þý­  ££SSSTVV⧟~‚D"8!у»]IØY€¥¥%ŠŠŠn»î&QoP*•HKKƒT*Å®]»PSSWWWˆD"„„„ÀÂÂBèˆDÔËXõÍÍÍÇ?ü€cÇŽÁÉÉ /½ôº¬ÛFÔ_ÔÔÔÀÊÊ ÍÍÍ]®×ÑÑ™™233aee%P:¢GssI(‘Hðí·ßªoÓÔÔĦM›ðÜsÏ ˜è7mmmØ»w/ÂÃñ{÷n455aáÂ…H$ âs ¢A‚!Q———‡ï¿ÿ?ýô„Õ«WcáÂ…ÐÐÐ:ÑC[·nÖ®]‹ööö[nÓÑÑ……>ÌÓÞ¨ßjjj‚··7ÒÒÒº\¯©© [[[\¸pZZZ¥#ºUss3!•J‘˜˜ˆöövxxx@$! Æ :"õ„DDD}R©D\\6lØ€äädŒ;/¾ø"ž{î9˜›› è‘577côèѨ­­½ã×èèè`ĈÈÈÈÀرc{1Q÷¸ù´âßÓÐÐÀÎ;y=õY7nÜ@LL är9öîÝ mmmøøø@,ÃÛÛû7Ú9}ú4Œùûœ¨Ò:ýŸk×®áÿøìíí±|ùrèèè >>øË_þÂrŒÍ›7ãÆwýš¶¶6TTT ­­­½”Œ¨{Ü­~+×®] ÎkP_5lØ0<õÔSˆ‹‹Cyy9¾ùæÔÔÔ@$aÔ¨Qxæ™g ^Cö^Ö­[‡qãÆ!))©‡“ÑÃà!QpòäIlØ°2™ zzzxî¹ç°fÍ888¨Ûµ··ÃÎÎeeew,G´µµ¡¯¯wÞyo¼ñwå¦~å^åàÍvïÞ€€€NDÔ}ÊË˹\Ž£GÂÄÄË–-ƒD"Á‚ n{Ú¼B¡ÀСCÑÞÞMMM¬]»ï½÷—J!êC8AHDD$ÖÖVìر...˜5kN:…¯¾ú %%%øâ‹/XÒ€%•JïXjkkÃÈÈ}ôJJJðᇲ¤~¥½½¦¦¦~[kðnˆ––þö·¿õR2¢îaaa×^{ ‡Faa!þú׿âÔ©Spww‡µµ5Þxã 9r¤Ëïø¤¤$(•J@GG>üðCøûû£¾¾^¨¿ý'‰ˆˆzYII ~øálÞ¼ÕÕÕ Ä+¯¼‚ ¨Ç©T*Lœ8èèèP_¯¥¥…aÆáOú^}õU.„OýžB¡À?þˆuëÖ¡¤¤ºü›¿Ù¾}û°dÉ’ÞŒGÔíòóó!“É —ËqîÜ9ØÙÙ!44‰ÿüç?ÕåtdØØØ ..“&M09,‰ˆˆzMjj*6l؀ݻwÃÔÔ/¼ðÖ¬Y+++¡£õš¸¸8øûû«ÿ¬¥¥…áÇã¯ý+^~ùe ˜Ž¨ûµ··C.—cíÚµ(((€¦¦¦z’ ømjvöìÙ8r䈀)‰º×™3g —Ë!•JQXXmmíÛîX¯­­ üôÓOX¾|¹I‰¨ B""¢téÒ%œ;w¾¾¾÷üÚ††ìܹß~û-rssáââ‚W_}ÁÁÁÐÕÕí…´4ØÕÔÔÜõöææf´´´Ür}gqw7†††ÐÑÑy <ãÆCaa!´´´`dd„÷Þ{kÖ¬ÁСCè~ˆú›ŽŽìÞ½ü1Îœ9--­.Eá¡C‡0oÞ¼n{¼›ö›ššÐÚÚ «¯:t(EÔãÖ¯_>øàŽkÎvž†ÿî»ïbýúõ·]Ãp ¹Ó±·Scc# Å-×ßë¸;|øðAñßz B""¢P\\ ;;;¿í°ª­­}Û¯+--Å7ß|ƒ7¢µµb±¯¾ú*f̘ћqI uuuP(¸qã†úI~MM  ÑÐÐ…BÚÚZ´¶¶¢©© 7nÜ@{{;êëë¡T*¡R©P[[ à· ¤Î[ZZÐÜÜ à·ºót­ÚÚÚ>³êÍ/`Œ¡¡¡---hjjâüùóGGGXYYaøðáê‚ÂÈÈššš022‚®®.†  2&&&ÐÕÕÅСCahh]]]cÈ!000À°aÃîøóHÔ×ìÙ³k׮űcÇÔ“UK—.ÅÞ½{oùÚªª*”——ãÊ•+(//ÇÕ«WQQQªª*ÔÕÕu¹ÔÖÖ¢©©éóèèèÀÈÈFFF066V_ŒŒŒ`ii ØØØ`Ô¨Q°¶¶Æ¨Q£øͼ˜˜xÛ Â›iiiaÞ¼yˆŒŒT¯ãÙSjjjÔÇÓ›?¯­­EKK šššPWW‡––466ªÍÅ{ç±€ú¶›ÕÇþ›åBÑÑÑQOçwŸ;µ ¯¯===ÿwÌî<ÞvÞfbb===èëëÃØØzzz000€‘‘ôôô0tèP >úúú|ïcAHDD‚jjjB~~>JJJPZZŠÒÒRõµÎ'_ÚÚÚ6l´´´`bb‚Q£FÁÒÒ£G†ƒƒÌÍÍ{%kII æÍ›‡ââb¨T*lݺaaa]¾&''_|ñ¤R)FŒ×^{ «W¯ÆÈ‘#{%#=ºëׯ£¶¶V}©¯¯W¿øîü¼¾¾^}ûï¯oll¼çctNíü¾ôê|»r­sš¯³ .Ï€;O ÜkBèN“‚7‘wr§éÄÛ•œmmmˆŠŠ‚©©)Õ/;‹Ñ›ï¯¶¶ … ]ÊÔ{é,9†uá1|øð[>þþvŒ9’¥õª´´4|üñÇ8xð àƒ>€B¡À… ŸŸ .t™2:t¨º¨355Uÿî,öŒŒŒÔ/Î; wêöJ¥²Ë¦?« …¢KÑXSS£þüêÕ«¨¬¬DEE…úû455akk õÅÑÑŽŽŽ7nœúq‰êëëajjÚeíÁ»ÑÑÑ©©)âââ0sæÌ;~]uuõ-Çëû½Ü«Dï<.>zzz044T¿ Õùsõ ¥›‰‰É-ÑùÜönÿn·äÆÝÎ èèè@]]Ý-×w¾9 Ü™Ùù ©© ---¨­­UO=ÞëÌ]Þh¸Ýå÷oFcĈ055U?Ç¡ÞÇ‚ˆˆzMkk+233‘––†“'O"77………êEÛMLL`ii ccc¨Ÿ u>±iooGmm-***PQQ¡þ>SSS8;;ÃÙÙóçχ››[·¿ó\QQWWW\¹rmmmÐÔÔ„ .\¸mmm8pŸþ9öíÛ‡‰'âü#žzê)uyC½O¥RáÚµk¸~ýºúÒù窪ª.×ß|ûí6è|—¼³`ºùÅøï¯ï|Q¡«««~7]OOÆÆÆÐÕÕÔk쵶¶>ÒÏÄÍÓ–…F狘úúz477ßRàÞ\ÞÞ\ôÞ©p>|8LMMajjŠ‘#Gª?v^~›©©)OѤrñâEœ>}YYYêËÕ«WüV 8;;w)Þlll`aakkkA( ”——£¤¤eee¸xñ¢ºÈ,((@ii)€ßÊgggLŸ>Ó§OÇÌ™31eÊþœ RÛ·o¿åÍÔûµdÉ899áÚµk¨¨¨Àµk×PUU…k×®Ývñ~J(ccc :´ËÜï'âXpßÛí&.oþ¼¦¦æ–b¶sÊù^e­LMM1jÔ(˜™™©µfff077ïr……75ëF,‰ˆ¨G#::±±±ÈÈÈ@ss3œœœÔ/&OžŒ‰'ÂÚÚúúú÷}¿ííí¨¬¬D~~>Î;‡œœdggãäÉ“hooǤI“àãム  Ì™3G½¾Í說‚››.]ºÔåp ¬^½ÇŽCVV-Z„·ß~ÞÞÞôxtw­­­¨¬¬TŸfWZZªþX^^Ž²²2”••¡²²ò–‰CCÃ.ÅÎïËŸÎÛFŒѥ䩫ÓÍbmm­º$¾[|ýúõ[îÇÌÌL=ÙÜy:fçÇÑ£GcÔ¨Q°²²Ôåð`¥T*qæ̤§§#==‡Fee%ttt0iÒ$u‰6mÚ4Lž<•••prrê—“¬ (((À™3g••…S§N!++ õõõÐ××ǬY³°`Á¸¹¹ÁÕÕ•§!>>>HLL„¾¾>T*T*:::nùüv´´´`kk‹Ç{L=5{sYdjjª.ýn7¥G}_çµµµ¨©©QÀ7—ÂÇåÊÊJTUU¡¡¡¡Ë}ÀÖÖVýFŠ¥¥%¬¬¬0zôhŒ=Z}ߤ¸7„DDÔíêêê°sçNü÷¿ÿÅ©S§0räHøûûcéÒ¥˜?>,--{ì±›šš‘‘””ÄÆÆ"77VVV‹Åx饗àèèø@÷WSS777äççßR6ijjB__¾¾¾x÷Ýwïz* ÝŸ¶¶6\½zÅÅŸ|ù2._¾Œ+W® ¸¸W®\Q?Y¼ÙÈ‘#Õ…Œ¥¥¥ºŒ177ÇèÑ£»”œè¤GÕÑÑqKiXRRÒ¥´¾¹¤¾yÊÅÀÀVVV°°°À˜1c`kk [[[ØØØÀÎÎvvv,M€’’$$$`Ïž=HMME]],--Õî...ƒf¢N¥RáÒ¥K8qâ>Œ´´4œ;wššš˜9s&¼¼¼àãベ3gòµ~ª¬¬ ………(,,DQÈm¢Þ IDATQ‘úRRR‚+W®tYvCOOO]ÜX[[«×¶ì|3åæÉ0NñÑ´´´¨KÃòòrTTT¨×híœp.))ér¶õyvvv3f ÆŽ«þ8vìXN"‚!u£üü||þùçJ¥‰D‰D‚ùóç ¶£Z~~>"##±yófañâÅxë­·àããsÏï­««Ã¢E‹››{ǵs444°uëV<ûì³Ý}@jkkCQQ.\¸€ÂÂB«/—/_FYY™zWO==½.ÊÍ/"ÌÍÍamm sss–~Ôg©T*TTT ²²R]"–––¢¬¬¬K~ózN#GŽTÿ»·³³SnooN!öQ™™™ˆ‹‹CBBNŸ> xxx`É’%pssƒƒƒƒÐûŒêêj>|D||<.\¸KKKøøøÀ×מžžü½Þ‡´´´àüùóÈÏÏGQQÑ-e`発ºº3fŒúbmmÝåôxKKK®ÇL½ª½½())QoôÔyüíü·Ü¹4ðÛñ·óßï¸qãÔ'L˜[[ÛAQZ³ $"¢Gvþüyüïÿþ/~þùgØÛÛãõ×_ÇSO=uÛ„ÒÑÑ}ûöaÆ HHHÀÌ™3ñÑGÁ××÷¶_ßÐÐwwwœ>}ú® kÿ~-Bú픺ââb¨/‹í©ÿ{šššv)Bììì`cc£¾ÎÂÂBà¿ Qï¸q㊋‹QTT¤.̯\¹¢.KKKÕŹ¥¥%Õ…¡ƒƒƒúóY¦]aa!vìØ;wâüùóprr‚¿¿?|||ðÄOð˜pŸòóó‡ÄÄD¤§§cèС ÅSO=777Nö’ŠŠ üúë¯ÈÏÏG^^žúó¢¢"ttt@[[ÖÖÖ·(SX£G ,---]&__€WUUømÓGGGL˜0NNN˜0a&L˜GGÇ5ùÏ‚ˆˆZCCÖ­[‡/¿üöööøàƒ ‹ûüÄS§NaíÚµˆÅâÅ‹±aÃ899©oojjÂÒ¥KqüøñÛ.‚ý{ƒuŠP¡P //¹¹¹ÈÉÉÁ¯¿þŠóçÏãÒ¥KP(sssØÛÛÃÑѱK‘ÁI(¢ûwóäm熟C©TBCCÖÖÖ°··Ç„ àììŒ)S¦`Ê”)066ú¯0`( ÈårlÚ´ ‡†¥¥%$ žyæ8;; ¯ß«¬¬„\.ÇŽ;™™ [[[¬Zµ kÖ¬á›Fݤ¾¾ÙÙÙÈÊÊBvv6rss‘ŸŸ¯žd611““&Nœ'''899aÒ¤I;vl¿\“èQÔ××ãüùóÈËËC^^žúó‚‚( hhhÀÆÆNNN]6gê¯kɲ $"¢‡²wï^¼ôÒK¨¯¯Çúõë±zõê>_ þ^FF^yåœ={ùË_ð?ÿó?hoo‡··7>|×ÉÁN:::ê¯kkk#*• ………ÈÉÉÁÙ³gqæÌõ Š¶¶6 2“&M¤I“ÔE`gÈb‚¨g) âüùóêÒ0//999¨®®ØØØ`Ê”)˜:uªº4œ4iOã|•••Ø¸q#¾ûî;\¿~ÁÁÁXµj/^,Ø]^^vî܉M›6¡¦¦"‘o¼ñ×û}ÅÅÅ]vËÎÎÎFaa!T*,,,Ô›ãt'N„¹¹¹Ð±‰ú<¥R©>öv¾Až““ƒœœ455AWWS¦LQ†›Qõ¥³«n‡!=¶¶6¼÷Þ{øâ‹/ ‘Hðå—_öë'“íííøî»ïð׿þÓ§OÇ¥K—P^^®¾]KK ÚÚÚhkkS/t¬¥¥KKKŒ?êÓmÄT\QQŽ?ŽÌÌLdff"++ ÐÔÔĸqãàììŒÉ“'cêÔ©˜:u*ìííd1JÔß•––"77gÎœÁÙ³g‘““ƒsçΡ¹¹ÚÚÚprrÂìÙ³1{ölÌ™3S§NåÏò•á“O>Á–-[0lØ0¬^½øÃ0zôh¡£ ­­­J¥øæ›opúôi,X°ëׯ‡«««ÐÑú”ÆÆFœ8qéééÈÈÈ@ff&ª««¡¥¥8;;ã±ÇS—œÈ$ê~J¥ùùù]Jùììl”——CCCãÇÇܹsáêê 777Lš4©O X° $"¢ûV]] ???œ9s6lÀ3Ï<#t¤n“““ƒyó桾¾`aa±cÇÂÁÁA½»Yç:;VVVfb¤¦¦F]v^*++1dÈLŸ>³gÏÆc=†©S§bÒ¤I000:2=¥R‰K—.áÌ™38sæ Nœ8ÌÌL\¿~úúú˜1cF—ÒpìرBGDmm-þùÏâ믿ÆÈ‘#ñþûïãÙgŸ…žžžÐѵÔÔTüíoCZZüüü°~ýzL™2EèX‚(..Æ‘#G‘‘£G";;íííptt„««+æ΋éÓ§cêÔ©êÞ£ª««Ã””„½{÷âÊ•+Ð××Çã?WWW¸ººÂÅÅfffBG%¢{P©TÈËËCFF†º0ÌË˃¦¦&f̘OOOxyyaöìÙ½:”À‚ˆˆî©¤¤óæ̓¹¹9âããûõ)Å÷¢R©ð§?ý _~ù%víÚ¡#=’¦¦&9rÉÉÉHNNÆéÓ§¡¥¥…9sæÀÝÝ...˜={6× $¢.ÚÚÚãÇ#55Äõë×agg§. /^< NS,**š5k°ÿ~¼ùæ›øŸÿùþnìÃT*~þùg¼õÖ[Ð××Ç?ü///¡cu›ŽŽüòË/HJJ¾}ûpìØ1hkkÃÕÕžžž˜?>f̘Ñ/7B ¢[UWW###ÉÉÉØ¿?rssabb‚Å‹ÃÓÓžžž°¶¶îÑ ,‰ˆè®êêêàææ$''cèСBGê}ô>ûì3ÄÇÇcñâÅBÇy ¥¥¥ˆŽŽÆ®]»päÈ´µµaúôéX¼x1ÜÝÝáææ6hþ?Q÷èèè@VVRRRœœŒC‡¡±±“'O†ŸŸ‚ƒƒûõæR©/½ôììì°yófÌ;WèHtŸªªªðÆo@*•bÍš5øúë¯ûí¼R©DJJ ¤R)âããQUU{{{õ4ÑÂ… yº0Ñ QRR‚¤¤$$%%áÀ¨««ÃÔ©S±lÙ2¬X±ŽŽŽÝþ˜,‰ˆèŽT*üýýQXXˆƒºÓVþð‡? <<gÎœéó Ò—””`×®]ˆŒŒÄ‘#G`dd„€€øøø`Ñ¢E9r¤Ð‰hQ(8~ü8öîÝ‹ÈÈHœ?ãÆÃòåË‚ÇBǼ'•J…>øÿûßñöÛoã“O>áîÎýÔîÝ»gggDEEõ«ç,'NœÀÏ?ÿ ¹\Žòòr¸ººB$ÁËË öööBÇ#"µ··ãèÑ£ˆ‡\.Gqq1fÍš…+V@$ÁÒÒ²[‡!ÝÑ·ß~‹÷ßÙÙÙ3fŒÐqzR©ÄÒ¥Kû÷ïïS»Œ¿­'¸cÇÈdÿ½ûŽ‹âZÿþ¡ Hg)"E+رÄX0‰%˜Ø\Qb‰^½^£Æ“k⢈]cC#؈5Š¢¢Æ‚ *(*ˆ´E–&u÷üþð»óE@Üe`yÞ¯/–awγ;sæÌ>sæœCøçŸ`dd„‘#Gb̘1pww§ÛŽ!õæþýû AHHbccakk‹1cÆ`ÆŒhÛ¶-ßáU©´´ß~û-Îœ9ƒíÛ·câĉ|‡D>Ò£G0bÄ”••áüùó :¹–žžŽ;v`ß¾}HHH@ûöí1~üxŒ?¾IžsBjG*•"""AAAFnn.ˆiÓ¦á믿þ¨óJB©Rrr2Ú¶m‹íÛ·cüøñ|‡Ã›—/_¢S§NX½z5¼½½ùÀ›ž›6mÂáÇ¡¡¡±cÇÂÃĺº:ßáBš¸ØØX„„„`ÿþýHLLÄ'Ÿ|‚Ù³gã믿n0Ç(‰D‚)S¦àï¿ÿFhh(ÝR¬Drrr0zôh<{ö W¯^Uø˜]*::¿ýö‚ƒƒ¡¯¯©S§b„ pqqá;4BH#SZZŠ³gÏbß¾}8qâLMM1kÖ,Ì™3FFF¼>JB©Òĉ‘ŸŸ'Nð ïöìÙƒE‹!11‘×YÃÂÂðóÏ?#,, ]ºtÁ¬Y³ðí·ßBWW—·˜ˆüååå)ýlœ Å«W¯põêUÄÆÆbÉ’%µ.ÚnïbŒáâŋضmŽ= kkk,\¸Ó§Oç}VàÙ³g#((çÏŸG÷îÝy…Èßëׯ1pà@¼~ý—/_®Óey‹‰‰ÁŠ+püøq¸ººÂ××ãÆ£[Ú›8y¶C¤i{ñâ±mÛ6”——ÃÏÏ~~~4ÙVúWŠBHƒp÷î]>|k×®å;”aÒ¤I°´´Äÿþ÷?^ÊŠŠÂçŸwww¨ªª",, ‘‘‘ðôôl´ÉA±X @€cÇŽÉ}ÝR©?ÿü3–.]Š>}ú ]»v¸ÿ>œáéé)÷ò䡸¸ëÖ­«õx‘ï{¤ö=z„_ý#GŽÄž={ê´Žòòr…B 4@Î*¶žÔ 4þù'ñå—_bÑ¢EpvvÆÁƒÁW?…#GŽ`×®]8zôh½$ÂvlÇŒúütttpòäI€ÂË«ŽX,†ºv튧OŸâøñ㈌ŒÄÔ©SUr°ªí×ÛÕÆ@íÐÇŠˆˆÀòåË¡¢¢xzz⯿þâ% é«äÉÚÚkÖ¬ÁÓ§Oñý÷ßcãÆhÛ¶-öîÝ[ûö–B!o™8q">|8ßa4(»wïfÆÆƬ¨¨¨ÞÊ,..f‹/fªªªÌÍÍ…‡‡×[ÙŠ–ŸŸÏ°?þøCîëþõ×_™¹¹9“J¥,''‡ 2„]¹r…}òÉ'láÂ…r/O^ÊÊʘ……«ÍéÙûÞ#ù0ååå srrú¨uXZZÖj»}(EÖ¾¼|ù’MŸ>©ªª²O?ý”={ö¬^Ëõê³°°`ëׯ¯·2뺟>}*·Â1ƒýùÑ£GLKK‹‘[ù å˜ÁÇþüóÏ?3VXXX¯å …B¦¦¦Æ<< :t(Ï‘4,Í›7Ç°aâð²?~ŒÞ½{ƒ1†{÷îaÆŒPQQQx¹õ­cÇŽèر£Ü×ûüùs¹¯³¡i ¡¨z·®]»âÎ;˜0aFŽ‰(¼ÌäädìÞ½¿ÿþ;ÔÔÔ^^E²³²²0lØ0dffÊ­ü†rÌàcž?>`çÎõVæºuë0þ|¬_¿‡þ 1À2e<)¢¾5t¬jzôè»wï¢K—.8p`µ·lS‚BH%×®]C·nÝ ©©Éw( N¯^½pýúu…–ñúõkŒ=¸rå ìììZŸæ͛ǡ˜™™ Ì›7ßÿ=z÷î™3g"-- Œ1œ:u >>>hÕª’““1xð`¨««£cÇŽˆŒŒœ|::: zï:+ÖeÓ¬Y3bõêÕ˜>}z­?çºÚ¹s'œœœÐ«W/…–S•yóæ¡yóæµÚ·ÃÕq™÷í›Õ‹¶mÛVåq±  >Ĉ#°|ùrxzz¢[·n¸víW^Mõ¸.u…ýYSS“'OÆöíÛ를ððp,Y²{öìá}üCy“m?‰DR©]ýãö‡ì7Œ1ܾ}K—.…ƒƒbccѧOhhh ]»v8uêTmqMmÁûê[mÚië>”¼ÚÍÚ¨íúèXÕxèëëãèÑ£0`<<<Þ¾©èB—ž={² ðFƒôÏ?ÿ0 3kåÊ•ÌÞÞž‰Åb…•ÑÐdff2[[[¶fÍn™X,fmÛ¶eVVVìÅ‹L$1ccc€ýôÓO,--…‡‡3Ö¹sçJëC·êäåå½³üçŸfšššÜx;Û·ogØĉ¹ç8::¾sËlYII ‹ŒŒdúúú  …,<<œyxx°¬¬,6tèP–––ƽî›o¾aFFF,''‡•””°^½z1oooîÿOŸ>ešššµºÅ§â{©.ŽGUûÙ¦¦¦2‰DÂÒÒÒfddÄ.]ºÄÒÒÒ˜††³²²bB¡³„„¦®®ÎúõëWmlùùù¬sçÎlƌܭñÛ¶mcØ¡C‡äº½ííí™ ÷·——ËÈȨ±ŒÔÔÔ*?K‰DRív{Ù­Y ,`ÿüóÛ³gÓ××gìîÝ»Œ1Æ/^Ì°¸¸8îuIIIlÔ¨QÕ~žMÅ¿þõ/Ö©S'…§Ð­[7¶hÑ"…­¿&R©´ÖûöÛÇ«êöÍŒŒŒ÷²³³«\cŒµlÙ’µiÓ†‹ÍÒÒ’9880Æj®Çu­+|¹yó¦ÂÛoÆÞl''''6þ|…–ÓTlWk»oè~S^^ÎΟ?ÏíÛ .dÑÑÑìèÑ£ÌÐЩ©©±ëׯTÈØ»õ£6mÈǶuïk‡ª"ïó¤ªT¼Å˜ŽUÊ«  €999±~ø¡ÊÿS‚BH%VVVì·ß~ã;Œéùóç û矲þׯ_3###¶wï^…¬¿¡š?>ÀD"Q¥å‡bØìÙ³cU'ëìí홊ŠJ¥eU\J¥Òw–÷ïߟ©««³²²2ÆØ›É °:pÏ©jLž·—Éâzýú5·ìôéÓ @•?GŽa7ndXlll¥u·iÓ惄ÕÅQÛ϶ªÏÇÁÁ¡ÊÏ[[[»ÚØ~øá€%%%qËŠŠŠX@@ËÌÌ”ëö644dX`` “H$,66–åææÖº Æ*–5m·÷‘í¥¥¥Ü²Í›73lüøñŒ1ÆÒÓÓ™––›1c÷œU«VÕëø{ Yvv6ÓÖÖVØçQZZÊ455ÙÁƒ²þQ›}ûíúX›}³ªcÀûÖÇc¿üò ó÷÷gŒ½ùR_1†šêq]ë _ŠŠŠ˜šš;~ü¸BË9yò$ÓÐÐ`YYY -§!¨ªÝ¨iß®ë~#[¯¬½fŒ±ÀÀ@€Mš4©ÒsêÒ¾ý>jó:yµuU•ÿ6yŸ'U¥ªó:V)§;w2@PéœE†n1&„RÉ«W¯`llÌw ’©©)€7c®(Bll,rrrЧO…¬¿¡º|ù2ÀÀÀ Òòþýû""" Êq544À«±Œª^Û§O”——ãܹsÀÝöéîî^ûà+¬[[[›[vãÆ ¸ºº‚½¹[égôèÑ8zô( uëÖ•Ö¥ªZ÷S³ªâø˜ÏV]]ýe(**ª6Ž3gά­­¹eZZZ˜3gLMM庽…B!ÔÔÔ0{ölôèÑ999Ð×ׯuo«i»ÕDCCƒ{üÕW_bbbæææðôôÄÞ½{‘šš Æ.]º„/¾ø¢Æõ6ÆÆÆhÛ¶í{·ÍÇ‹Å(--…¹¹¹BÖÿ!êr,«Í¾YÕ1 :‹-ÂäÉ“! €’’.†šêñÇÖ•ú¦¥¥###¼|ùR¡åܺu mÚ´@ Ph9 AUûqMûv]÷Ùz+¶KÇðf¨‰ŠÏ©Kø¶Ú¼N^m]mÔÇyRUèX¥œzôè‘H„„„„wþG BB!„ðJ*•ž={Vi¹,Q­£££rW¬XüS§NÅþóøúúbùòåøå—_>z݉‰‰‰())©ò²½ksðcññÙÊÞó“'OÓ”)Spûöm 4‘‘‘èÓ§Ö¯__ç2jÚnB– ¨89Á÷ßÆÖ¯_Û·oÃÍÍ­ÊD,!o“ç¾)sñâE8::ÂÕÕ¾¾¾•ÆÛª©+"¢üä¹ßXXXx3–éûÔµ-¨Íëê³­ãë<©.èXÕ¸Q‚BH%&&&xõêßa4H²žƒ²ž„òÖ®];+¬MC5hÐ àzòɼxñðå—_*¤\©TŠÜÜ\ܹs?ýô<ˆ•+WV9AOÅ“HYOÃê´oß……… ¬´<-- °··ðî{–7>>Ûîݻ֬YS©—H$‘#GäÓ/¿ü‚Î;ãÂ… 8zô(TTTðßÿþ·ÎeÔ´Ý>„¬§ÒÈ‘#¹e666˜8q"¶nÝŠ€€LŸ>ýƒÖ©Ì²³³‡O>ùD!ë744D³fÍ‘‘¡õ+BÅú#Ï}SfêÔ©ÐÕÕåz!U,¯¦z¬ˆx©¨¨¯^½‚¥¥¥BËéÑ£!‰ZNc%Ïý&''@õ½þ?¤-¨¸Ÿ×æuõÙÖñužT[t¬j\nݺ@GGÇwÿùQ7/BQ:4IÉûÉ&)yþü¹ÂÊMRòêÕ+…•ÑЈD"æààÀZµjUiÀèE‹±Î;³‚‚ÆØÿŸ¯â$²qòdËJKKÖºuëJe”——3Ü ×Œ1¶dÉÖ¶m[¶wï^væÌvíÚ5öàÁnÒÆ:t(À6lØÀ’““Ù¶mÛ¸A»###Yyy9Wyy9÷º‚‚fccÃTUUÙ‚ Xhh(Û¸q#stt¤IJ!„ÔÎœ9sjœ¥´© …ÌÂÂB¡e¼~ýšuêÔ‰õë×åææ*´¬†$33“Íš5‹õêÕ‹}ÿý÷ÌÇLJ-Z´ˆååå1Æ;pà7Ãï¦M›Xnn.Û»w/SSScØÚµkÙ½{÷ØÊ•+¦¦¦Æ6oÞÌâââXAA[¿~=À455Ù¾}ûX^^;~ü8w’YñGOOíܹ“1ö&ÙÕ»wo¦¦¦Æ:tèÀnݺÅ>ýôSæååÅöîÝËÖ®]Ë444¶|ùröðáCî=ÅÇdzÏ>ûŒiii16iÒ$–žžÎýÿÌ™3¬{÷îLCCƒ™˜˜°… ²¾}û²™3g²°°°wNœeë|û=Þ¹s‡­_¿þ½qÔôÙfff²_~ù…{ïW¯^eW®\aÚÚÚ [³f ËÎÎf{öìá¶ÁÖ­[«Mb>xð€}ùå—LWW—ikk³±cDz—/_Êu{qƒ™¯]»–ýûßÿfC† aOž<©UÏŸ?g«W¯æö‹={ö°œœœ·[UN:Åh”3 IDATÜÝÝYÏž=™§§'›:u*[¶l+,,¬òù£Fjr“UçÇdÍš5c·nÝRh9?üðk×®B˨Im÷í-[¶0]]]æããýö}û¦ìWÕ1 ªc†l&íÀÀ@¦««ËìííÙ™3gØŠ+˜ºº:ëÓ§KOO¯±×¥®ðeéÒ¥ÌÅÅ¥^ʺrå SWWg»wï®—òøPU»ºeË–ZíÛuÙod Â7²ÜÜ\–’’ÂV¬XÁÒÒÒªÝÿ«¹-`ŒUYßjóºmëÞ×UE^íæÛ"""زe˸s ©S§²ÐÐP:V)¡’’öÕW_1ggg–ŸŸ_åsT«ãˆ•„B”Ò¡C‡0}útäääT;®KS4nÜ8”••!$$D¡å$%%aàÀ000ÀÑ£Gáàà Ðòš"ÆÀƒ¯¯/·¬¨¨çÎäI“>> iš¤R)úô郰°°5nŠ‹‹1wî\ìÚµ ûöí÷ß~«Ðò’““áàà€‹/*ìVfÒð”––ÂÖÖK–,O½”) 1þ|üþûïðóó«—2•™³³3âããë<Ù!MX,ÆøñãqëÖ-\¾|íÛ·¯òy4!!„J† œ>}šïP”‚‚œ:u _ýµÂ˲··Çõëס¡¡lÙ²… šÈÇòåËáëë OOOn™ŠŠ tttУG´jÕŠÇèˆ2Û¹s'úöíÛ䓃·oßF×®]ñçŸ"44TáÉAàÍžžžX¸p! Nß„¬[·˜1cF½•éçç,\¸cÆŒ¡± !¼ùçŸÐ¥KÄÄÄàÒ¥KïM” $„ò}}}|ýõ×عs'ß¡4(ÁÁÁÐÔÔĨQ£ê¥R©çÏŸÇÖ­[ ;;;,\¸Ó¦Mƒ¦¦&¯±ùøøà?þÀ… гgO^c!òWPPwww¼~ýW¯^…ß!!..+W®Dpp0Úµk‡¹sçb„ ÐÖÖæ;4Bˆxþü96mÚ„mÛ¶AMM ~~~˜7otuuk½JB©Rrr2Ú¶m‹mÛ¶a„ |‡Ã›ÔÔT¸¸¸`õêÕðööæ;@tt4pèÐ!¨¨¨ÀÃÃ4hï_º !$&&GŽÁþýûñôéS 0³fͨQ£ ¦¦ÆwxÞ )0iÒ$œ;w'Nœ@ïÞ½ù‰ÈÉ«W¯0räH¤¥¥áÒ¥K°¶¶æ;¤J>|ˆß~û „®®.&Mš„‰'¢K—.|‡FidJJJpúôi®— ¥¥%æÌ™ƒY³fÕ©% !„¼W`` –,Y‚¨¨(ØÛÛóN½“H$pww‡ššþþûo¨ª6¬‘9rssqàÀ:t×®]ƒFŒ¸»»Ó,Ô„z…9rñññ°··Çرc1mÚ48::ò^•JKK1yòd?~[¶lÁÔ©Sù‰|¤ØØXŒ1Œ1œ;w­[·æ;¤÷ÊÊÊÂöíÛ±ÿ~ÄÅÅÁÙÙãÇÇøñãáààÀwx„J*•"<<ÀÑ£G‘ŸŸÁƒcÚ´i=z4ÔÕÕë¼nJBy/ÆFŽ‰G!""¢É›5{öl#&&-Z´à;œj¥¥¥áèÑ£ ÁÕ«W¡««‹áÇcØ°a8p ÌÌÌø‘¢DŠ‹‹qãÆ œ;wGŽÁãÇѺukxxx`̘1èÚµ+ß!Ö c +W®ÄªU«0wî\üüóÏÐÒÒâ;,R!!!˜1cºté‚àà`¾Cªµ¨¨(î‚_jj*ÜÜÜàáá¡C‡ÂÙÙ™ïð!<+--EDDN:…C‡!-- nnn7n¾ùæ¹çS‚BHµrssáææ†fÍš!""âƒÆ±hÌ–.] œ9sýúõã;œ’™™‰£Gâرc¸zõ*Š‹‹Ñ©S' 4ħŸ~Úd¶#!D>$ "##qñâE„……áÚµkܱeøðáøúë¯áââÂw˜u OOO´hÑ»ví¢[Ž‘ÌÌL|÷Ýw Á¬Y³  ¡¡¡ÁwXu"•JqùòeáäÉ“HOO‡­­-¾øâ |ñÅ4hµß„4Ïž=ÃÙ³gqöìY„……¡°°®®®=z4Ư»»(AH!ä½nܸ¡Pˆ£GBGGŽŽŽ8yò$ÌÍÍùMacX°`6lØ€£GbĈ|‡ôQd½|†;wîzö쉢W¯^èÑ£LLLxŽ”Ò””” ::7oÞÄ¥K—±X {{{ 4ˆ»à L=ËSRR0sæLœ;wsæÌÁªU«`hhÈwXä=¤R)öíÛ‡ @OO[·nÅgŸ}ÆwXrÃCtt4— ¸~ý:TUUÑ·o_|þùçèÛ·/ºvíJÉ¢$²²²pãÆ „‡‡ãÌ™3xôèŒ1xð`î"………Bc !!„JÊÊÊ¡Pˆ[·n¡{÷îðó󃛛†Žââbœ:uJ)oyyýú5¦M›†¿þú AAA9r$ß!É]^^._¾Ì% ccc!•JѺukôèуûéܹ3ÝfGHÁC||—lˆ‹‹Cii)455Ñ®];´mÛmÚ´££#Ú´iƒÖ­[ÃØؘïÐ Qj%%%HLLÄãÇñøñc$&&âÑ£G¸ÿ>rrrvvvèС:tè€N:¡}ûöpvv¦Äþ‰Dغu+‘••…Ñ£GcêÔ©E\\âã㘘ųgÏ””„gÏžq?²žøººº\¢¼mÛ¶•çÊ4‰% !D‰]»v þþþ8vì,,,0gÎxyyÁÄÄD!å=~üëÖ­Ã •JñÍ7ß`üøñèß¿?o'qqqÆ®]»’’wwwÌ;—ƒ<‘H$HNNÆãǹDÈóçÏñüùs$''#55eeeMMMXYYq DkkkXXX E‹Üï-ZФ¤Á*//GFFÒÒÒžžŽ´´4î§â¾_PPÀ½ÆÜܼR²¼U«VhÕªìííáèèH³‹7PQQQ Å©S§pçÎèëëcðàÁpwwG¿~ýжm[¾Cl0233K—.áÔ©Sxúô)¬­­ñå—_bøðá4h%»ÒÒR<~üñññ\Ò$))‰K¨ÔÕÕѲeK.yزeKXZZÂÒÒVVVhÑ¢ÌÍÍ)áKêMII ÒÒÒššŠ—/_âåË—HMMå.Æ%%%!++‹{¾¹¹9—ð–%¿íííáää„–-[òøNê% !DÉÈn …¸}û6zö쉹sçâ믿®·ñ¦òóó„?þø7oÞ„±±1FŒwwwôïß––– +»°°ׯ_Gxx8Ž;†¸¸8ØØØ`ܸqðòò‚ƒƒƒÂÊ&O"‘ --K¾ý“™™‰ÌÌLH¥Rî5zzz°²²‚™™¬¬¬`nnÎ%ÍÌÌ`jj @^g†#Ê¡¬¬ "‘ÙÙÙÈÎÎFFFF¥ä_FFRSS¹}µâ©¶¾¾>¬¬¬`aaÁõ”­˜ ´±±¡ HOOÇÉ“'qöìY\ºt ¯^½‚©©)>ùäôë×nnnèرc“¸¸!•J‘˜˜ˆÛ·oãÊ•+ˆˆˆ@\\455ѽ{w 2Æ ƒ««+ß¡’:ÊÊʪ²ç•ì¢_nn.÷\ ´hÑÖÖÖhÑ¢¬¬¬¸$¢¬ÍÔ—¼WAA²²²™™‰¬¬¬JÉ¿´´4¼xñiiiÈÌÌä^£ªª sssXYY¡U«V\"P–´³³Sªž€uE BBQPÏÌÌĘ1càçç777^ãzñâŽ;†'Nàúõë(**Bë֭ѵkWn|¬¶mÛÂÚÚúƒ’7eeeÈÈÈ@BB>|ˆ ::wïÞ…D"A‡0tèPŒ3ݺu£+ÖJ¤¼¼™™™HOOÇË—/‘‘‘Áý–%fRSS‘‘‘âââJ¯ÕÒÒâ’…¦¦¦011áþ–=611¡¡!  ¯¯O½Y”T*Enn.Äb1òòò ‹‘ÌÌLdggWJfgg#++ "‘yyy•Ö£¦¦333î‹GÅDµ¥¥e¥„5}ùhz¤R)bccqùòe\½zW¯^ÅË—/¡¦¦†¶mÛrƒ×wéÒNNN ½€¦hyyyHLLĽ{÷…»wï"&&hÞ¼97sg¿~ýгgOªMÄëׯñâÅ ¤§§#%%ééé\§bÏ®·ÛkCCÃJù,,,¸Ç²6ÜÈȈk¯úØä]EEE\[œ““‘HÄ%ÿ233!‰ ‰*ý]TTTiFFF•zªÊî8iÙ²%,,,`mm sssº¾(AH!܃°aÃìß¿ÚÚÚðòòÂœ9sdWøÒÒR®Add$bbb””‰DàMO0kkkèëëCOOÐÕÕEII ^¿~’’äåå!##\ÏssstìØ:uB¿~ýзo_…ÝFM—¼¼¼*>²NÙOÅDì犚5kÆ% `dd}}}î ¡¥¥mmm.¹¨§§4kÖ FFF<|Ê¡  ¥¥¥‹Å(..FQQòòòPRR‚üü|"//yyyÈÍÍE^^rrr¸Ç—çççWY†‘‘Q¥ÄqÅä±™™Y•ËéK)ùÉÉɈŒŒä&‰¸{÷.^¼xàÍ8Wo1ieeÅõ´âóøQ\\\©§NRR7qN||<×[G__...èÒ¥ 7ƒgûöíéË9©VNN×#L–$ÊÈȨ2A”••Åg\‘žž ¹„¡ìñÛ yóæÐÒÒ‚ttt ­­Í=¦‹‚5ËÏÏGqq1òóó+=.((@QQÄb1Äb1rss‘““Ãý-[&{,›Ô®"]]]˜™™q=JMMMajjZ)aljj sss˜ššÒÅ9¢!!„4B² 7„B!ÂÂÂàä䄹sçbòäÉî–¥ââb$$$ %%…»šœ——Ç% ¡¡¡ððp´k×={ö„¹¹9w5ÐÉɉ’D®rss‘Íõ,“%”*&™*&ÞN>U¼ê}455ѼysèééASSÐÖÖ†––÷?•¾¨èëëCMM ***Ü Ð²$:îõU©)©PÕÿ «üVÓÿ% ×ËN–ÜÀ%ù7‰[‰DÆÄb1€7½‚ ¸×¡¸¸b±¥¥¥•Æé{w’¶{‚Ê~¿ý?ÙcJö¾ˆD"ÄÅÅáñãÇHLLäo‰‰‰ÜoÀ›z.ë%#¸ý·â~¬§§ òñ£Y³fÐÑÑAyy9—¯XÿŠ‹‹¹ãWÅ/ï999Ü-ô²Yn7ã͵jÕªÒìôŽŽŽhÓ¦ ììì¨×>Q¸¼¼<ˆD¢w’Oo' ªúûížàoSUUå’…ZZZ044äêSÅz%k;eËÿßV«««su±yóæUÞ2]]2RMMíè+¶¯U)((¨ò"§l¹¬ç<®±X ÆwÞ ¼IÊŽEEE(((¨”¬ŽššZ¥ä¬‘‘Qµ [ÙßÆÆÆ4Ô(AH!HAAöìÙƒ 6 11Ÿþ9æ΋Ï?ÿ\éOÆÛ·o¬X±‚ïP©QUÉ­ÂÂB”””pWÌ_¿~¼¼<”––"//K ÉzÆ•Oöe'ðïKÀՔЫ/µI`êééq½‰ ¡¢¢Â}’}1“õÂ400€¦¦&ôôô¸/YFFFÜ2===4kÖì/R„(‹œœœwÆ×JOO‡H$ª”Ô“ýÔôå½*²^Òç²[7e·ÈW/ÎÜÜœf™'ZAAŠ‹‹‘——‡ÂÂB®GúÛeÉsY+û]—d_>&©©«« mmmèééAWWZZZÐ××ïã÷%CIã@ý¼ !¤xþü96mÚ„íÛ·£´´“&MÂñãǛԬˆæææÈÈÈà; BjE[[ÚÚÚ âvbYϼ÷©ø%ÆÁÁëׯLj#*õd¬JMÿ'„ȇ‘‘ŒŒŒÐ¾}ûzìÃĉ¡­­_ý•KÀËÈô„45ºººÐÕÕ…@ à-†ÜÜÜJ“®UTñba`` Ž;†óçÏÃÀÀà½ÉyÙ…5Bꊄ„Ò€EDDÀßßÇŽƒ¥¥%þýïÃËË ÆÆÆ|‡Vï²²²øƒFGCCフfff°··W`D„ú ë½[XX{{{ª×„40“õÕ155Eyy9Õa¢pt‹1!„40¥¥¥†P(Ä;wàææ???Œ=|‡Ç___DGGãÊ•+|‡B!„4*ÅÅÅ4®!TII MœBêõ $„B$aË–- „H$˜1c€ž={òZƒ ¸ !„R{”$¤ñ¢ä ©/” $„žÝ¿6lÀþýûѼysxyyaÎœ9°¶¶æ;´ÅÜÜ"‘ˆï0!„B!DéP‚Bx •Jqúôi…B„……¡mÛ¶ …˜4i7£©ÌÔÔÙÙÙ(++kÒ·ZB!„Bˆ¼ÑÜô„Ròóógggn–ÐsçÎááǘ9s&%«affÔ‹B!„BäŒzBH=xöì°cÇ”——cÒ¤I …³³3ß¡5²aFFZ´hÁs4„B!„¢<(AH! tõêUøûûãøñã°´´ÄÒ¥Káåå###¾Ckt ++‹çH!„B!D¹P‚B䬴´þù'üýý‰^½z!((£G†º:vëÊØØššš4“1!„B!„ÈAH!r’••…ü­ZµÂŒ3àèèˆ[·náúõë;v,%åÀÄÄ„„„(ŠŠ ‚‚‚øƒ"'!!!4±!\ll,ÔÔÔpÿþ}¾C!JŽ¾­BÈGŠ‰‰¿¿?‚‚‚ «« ///Ì™3VVV|‡¦tZ´hA·B!µ$‹¡««Ëw„ ­­ ©TŠ²²2¾C!JŽ„„RR©§N‚P(ÄÅ‹Ѿ}{øûûcâĉ4±™˜˜ ##ƒï0QZ°±±á; BˆœXZZbìر|‡AùÍ›7ÇÈ‘#é;Q8Æã;Bi,òóó±{÷nlܸOž<Á!Càççwww¨¨¨ðžÒ›}Š€€ìر‰S¦L¯¯/œœœø­IHHHà; B!„BQ*” $„j\¹rB¡¡¡¡°²²Â²eËàååCCC¾Ck’ÌÌÌ ‰øƒB!„B” % !ä-%%%øóÏ?áïï»wï¢OŸ>8xð FE3óÌÌÌ ééé|‡A!„B!J…¾éBÈÿÉÌÌÄ–-[°yófdggÃÃÃ[·nE·nÝøüSSS¢¨¨ÚÚÚ|‡C!„B!J„„&ïÞ½{ð÷÷GPPôôô0sæLÌž=–––|‡FÞbnnÈÈÈ€­­-¿ÁB!„Bˆ’På;BáƒT*ʼn'0pà@¸ººâöíÛظq#RRR°zõjJ6P••Ås$„B!„¢<(AHiRòòòàïïGGGŒ=:::8þ¥¦¦ÂÚÚ×®]CïÞ½ù‡¥2vìXøúúÂÖÖöã([ i\V®\ CCCÌ;·N¯—w···‡¦¦&=z$—õÒ”;v ×®]úuëêôzªÏ¤6ècBïîÞ½‹)S¦ÀÖÖ›7oÆÌ™3ñüùsìÝ»÷ƒ“ƒYYY6l233-ᛩ©) ##ƒçHQ>ÁÁÁxðàÁ;ÇQ:¶ÒøDDD 66¶N¯UDþü¹ÜÖEHSààà:½–ê3©-JBx!‘HpìØ1ôïß]»vEtt4ñüùs¬Zµ uZo`` bbbžžooo9GMMMMB$áСCÐÒÒ‚ŠŠ ??;v쨴Œòa.\¸ðÎq´ªcëÁƒѼys¨ªªB(¢¼¼pøðaèèè ((ˆ·÷@ys®e``€Ã‡cÊ”)èׯc8uê|||ЪU+$''cðàÁPWWGÇŽ àýçSùùùXµj<==ѯ_?ôîÝ·nÝ‚D"ADD/^ $%%ÁÅÅ&&&ضm¼½½!•J¹õy{{£  >Ĉ#°|ùrxzz¢[·n¸víW^aa!V¬XÉ“'cÞ¼yèÙ³'V®\ ‰DRm<„(H$ªÏD±!„Ô£ÜÜ\öÛo¿1;;;¦ªªÊ†ÎÂÂÂäZæää$×u’†ÅÑÑ‘­ZµŠ{üvsVÕ2BHíUu­jÙâÅ‹Ç-KJJb£Fª—8 !5ËËËãê¯T*e"‘ˆ3짟~biii,<<œ©¨¨°Î;s¯{»ÎK$6tèP–––Æ-ûæ›o˜‘‘ËÈÈ`‘‘‘L__Ÿ`B¡…‡‡3–]åúc¬eË–¬M›6Œ1ƤR)³´´dŒ1ÆòóóYçÎÙŒ3˜T*eŒ1¶mÛ6€:t¨Úxrrräü)Ò0P}&ŠD“”BêÅãDZqãFìÞ½0mÚ4øøø uëÖE¢!!D¡.^¼œ>!!! …®®.6oÞ ---"88˜K*˜››ÃÔÔ زe ÂÃÑ›› 333èêêB ÀÐÐ/^Dhh(ÆŒ+++DDDàÎ;˜5kƇääd\¹r§N‚ŽŽ6oÞ “jã111áù“'Dþ¨>ESat!ääææbÇŽ@rr2†Ž¹sçbÀ€|‡Fš‹/bРAÈÌÌ„©©)ßáÒäI¥RôéÓaaaÐÑÑá;Bš¼’’”••ñzÁ–BHãBcBj%11>>>hÙ²%V­Z…¯¾ú 8~ü8%I½377î6cBˆ|¨¨¨ ((èƒ_·sçNôíÛ—’ƒ„4‡âÚJBHãÜ¿ŸïPˆ£{®!Õ ƒP(ÄéÓ§akk‹•+WbÆŒ41ᕬ×`FFÚµkÇs4„4MgÎœÁ‚ P^^ŽœœÄÆÆò!äÿˆÅbò!DN´µµyyy¢P4!!„#›F~ëÖ­ÈËËøqãàëë‹Î;ó!ï077Çÿû_|÷Ýw|‡B!„B!ÝbLAdd$Ö¯_àà`aÎœ9ðöö¦±kHƒ&h BB!„B‘JÒDI$;v þþþˆˆˆ@çαuëVŒ7Íš5ã;ˆ‰‰ % !„B!D(AHH‘€ 6`Ïž=PUUÅŒ3ðÝwßÁÞÞžïЩ“-Z **Šï0!„B!¤Ñ£!!JŒ1†°°0…Bœ9svvvX½z5¦M›F3“FÏÄÄ|‡A!„B!ž*ßB䯸¸Û·oG§N0xð`¼~ýGEBBæÎKÉA¢,,, ‰øƒB!„B=J¢D^¾|‰eË–¡eË–ðññA÷îÝ‹/⫯¾‚ª*Uy¢<^½z…²²2¾C!Di#99™ï0!¡¼¼›6m¢ºLˆ’ùûï¿qÿþ}¾Ã JŒ²„(;wî`„ °µµÅÎ;áããƒäädìÚµ ...|‡GˆB˜™™õ"$DŽÆŽ‹ˆˆ¾Ã „|±XŒï¾û?æ;Bˆ-Y²AAA|‡A”% i¤ÊËËŒ¾}û¢{÷îxôèvìØgÏžaùòå\ò„e%ÛÇÓÓÓyŽ„Bi8Äb1ÀÐÐçH!ò¤££ƒ‚‚¾Ã JŒ&)!¤‘‹Åؾ};ššŠ#FàòåËèׯß¡R¯LMMPBBäéðáÃèÙ³'ßaB>‚••.]ºGGG¾C!„ÈѪU«```ÀwD‰©0ÆßABj 6`Ïž=PSSƒ§§'¾ûî;ØÙÙñ!¼iÖ¬vî܉‰'ò !„B!„4ZÔƒŒ1† .@(âÌ™3pppÀš5k0mÚ4èééñ!¼ÈÊÊâ; B!„BiÔh BB ¢¢"lÛ¶ ;vÄgŸ}†ââbœ8qñññðõõ¥ä !ÿÇ™™™|‡A!„B!õ $¤IMMŦMÿ½;‹ªÜÿþ†]VEdT@QDPTÑ\AM-Ò2·´ô—KVšW­¼ÙbYz+óæRîÙÍLÅ%5•-MYÙÁdW†f¾¿?¼s.#æ gž÷ë5/pfœóÃy–ó=Ïyžؽ{7*++1gÎ9r|‡Æ0jÉÂÂ<@vv6 Q\\Œââbôë×£Fâ;<†QkGEyyy«Þ;sæLôèÑCÅ1 ÓV±±±X»v-zô蘙™ÁÌÌ æææJ711á;d†aZ!33™™™¨ªª‚X,FMM jkkQ[[‹ššˆÅbTUU¡®®#FŒ@hh(ß!3›ƒaÔ@LL ¶mÛ†cÇŽ¡GX¶lÞxã n†a€‹/b×®]¨¬¬DQQòóóQRR‰DÒè½ÇÇóÏ?ÏC” £9^{í5ìÝ»:::M¾G^¾Äbq³ïc†"‘ˆ[­XWWD©T ©TªðÞåË—ã믿æ#L†aÚ(<<cÆŒáþ­¥¥¡P@@J¥¨¯¯Ç±cÇ0sæL¾Be:– džÔ××ãøñãؾ};®]»///¬\¹¡¡¡ÐÕÕå;<†Q;iii­Z‘Q  ¤¤Ý»wFs]¼xãÇoö=:::˜?>víÚÕAQ1 ÓV¾¾¾¸~ý:Z:­KIIAÿþý;(*†až¡W¯^-ε­««‹ÒÒRuPdLgÆæ d˜öèÑ#|þùçprrÂœ9s`ccƒÈÈHÄÆÆbîܹ,9È0MpqqApp0´µ›ŸÃÍÍ%¦‚‚‚Z¼mX"‘àå—_î ˆ†y!!!ÍŽð …7nK2ŒXµjU‹eû¹çžcÉA¦Ý°!Ãt””,]ºöööøä“Oðâ‹/"==ÇŽƒŸŸßá1ŒFxûí·Q__ßä뺺º˜4iRFÄ0šK(â•W^iöÂT¯^½XÅ0jnâĉ‹ÅM¾.•J±|ùòŒˆa˜ö0oÞ¼FS4DDìÖb¦]±!èáÂ… ˜}úÀËË‹‡ˆ†i«É“'C&“5z^KK ‹-â!"†aÚCpp0·RyCZZZ˜5k1ËV0L víÚ…Ù³gþüóOäææ6zÏõë×1{öl899aÿþýXµjòòò°{÷n 4¨£Cf˜NM[[«W¯†P(Tx^WWÏ=÷OQ1Œæš3g$‰Âs:::˜7oO1 ÓVÏ=÷\£UŒuttðꫯ6{û1Ã0êMWW .lt!¯¾¾Ó§Oç)*¦³b B†iÆ–-[ðÆopWd…B!¾ùæWvüùçŸáëë‹#F ==?þø#²³³±aÃ6Ü›aTèµ×^k´°‚X,ÆرcyŠˆa4—››ÜÝÝ!¸ç$ ^zé%£b¦-,--1pà@…ç$ –-[ÆSD ô—… 6º7pà@6Ÿ=ÓîX‚a” "¬Y³ï½÷žÂÕX‰D‚;wâ£>Bß¾}ñòË/ÃÎÎQQQˆ‰‰ÁË/¿Üìj ô333,]ºTájªŽŽ|}}yŒŠa4×ܹs¹ùË<<<пž£b¦-BBB¸~¨P(Ĉ#àééÉsT Ã<+777 6Œ›®JGG/¼ðÏQ1K2̤R)–,Y‚/¿üRéë555زe BCC‘‘‘_~ù£Gîà(†Y¹r%7ºW `øðá000à9*†ÑL/½ôêëëÄÇñèÑ£&¯¨¨€H$R8Ñ~ôè×ahI[’o 5w2ÝðdYN^¹)#O4$ï¸(KT*{­9zzzèÖ­ÌÌÌЭ[7tëÖ FFF055…±±1ºwïssstïÞ]á÷†Ï5”ŸŸ   ddd´*9ØPRRÜÜÜÚôFÝÈ;(òúHþû“ÏÉ;=ÕÕÕÜ ùÅŠŠŠŠ·Ó°>’'è”%åžìÔàNN”ÉÎÎFZZšÂÕÔæŠ “‘ É;cò‹1Àÿ’•òÏkm=%ÿ>FFF044äê(ùïæææ uSÇü9vâÅ<«êêjüù矸té®_¿Ž;wîàáÇØÙÙÁÎÎŽ; …¸|ù2  !‰PSSƒ¿ÿþÿý7 ADÐÕÕÅÀáéé‰ÀÀ@Œ;ÖÖÖ<[†Q­ÚÚZ””” ¨¨EEE())Aqq1JJJPPP€²²2”——+´—åååM^€Ðè$]Þ&Ê/Ä?yA^YŸüÉçËË˹rü¤'Ï'ämšL&ãúö"‘ˆk'¶‡ÊÈ·knn®8433ƒ……zöì‰^½zq¿[XX W¯^M¶ç £ÊËËQXXÈ•ï’’…Ëýååå\™oîÜøßgy[+ïËÊÏÓåäåÿIúúúøí·ß0{öl‚&·+O@Ê5¼!¿€¯ì"ESôõõ¹òmbbsss.‰¨¬ŒËgÌ5K>ƒG¡  ÅÅÅÈÏÏç*Ž ¨¨ÅÅÅ(--å€O …WFFF033S8©T6ŠNž333S8éî,äÉCyr´ªªJéèÉÊÊJTUUq#–***>Tz»pÃý}ûömîy--­F•¬@ à*By¥gaaSSS¼ð lFíÔ××£¸¸EEEÈÏÏGqq1Š‹‹QPPÀÕMEEE(--Å£G”&¼A£ä•¼cÐp´œüwyÝd``  “ÿûÉ„_{’Éd8qâf̘¡²m(#O(6 "¯³ª««¹«ÇOŽª¬¬¬l”€­¬¬lôù::: íCÏž=amm KKKî¤KÞ ³²²âF0][AAŽ;†_ýW¯^E}}=<==áããƒÁƒÃÃÃîîîMž gee¡OŸ>J_“H$HKKÃ;w›7oâÏ?ÿDUUÜÜÜŒÐÐP :T•_‘aÚU]]—ÏËËøѵò>}aaa£zZOO°°°€••7ڦᨛ†I³†'ÕòQ:ª’ GGÇvû<‰DÂŽj˜ü”ÿÞ0a"‰ ‰¸¾Gsû®gÏž°µµåööö°±±Q¸xÁ0íE$)”í¼¼<®ìÿý÷ß\ðÉD™±±1×÷êÙ³'ÌÌÌÊ·pNNz÷îÝ®Ÿ)×°ûäÅŽ†¿ËË|YY ¸Dê“ûÎÈȈë¯6¼ÁÆÆööö°µµ…M“w&2%•H$xðàrss‘““ÃU$999(((àN²û …èÙ³',--¹8KKKôèÑCá¯á¨6Öèu ‘H¤0RSþ(,,Äo¿ý†’’H¥Rn$TÃ$¡žžwbneeØÙÙÁÞÞ½{÷æ*9U&@F®¨¨÷ïßÇßÿÜÜ\®~ÊËËã:ãÅÅÅ ÿ§á1Ü°ƒ#¯Ÿ”d333ãévMò "Mä,--UšämØ„ëêê*´C¶¶¶°³³CïÞ½¹“¯Þ½{«ô¤”á‡D"Á¯¿þŠï¿ÿ‘‘‘055ÅóÏ?)S¦ 00°ÑÈùö$‹qãÆ œ?ÇŽCJJ \\\ðꫯbÉ’%°´´TÙ¶¦5Äb1rrržžŽÌÌLddd ##¹¹¹\PNGGÖÖÖ '®½zõ‚¥¥¥Â++«f§±`ÉG_ʆòß‹‹‹¹äŒüÜ«á-ÔݺuƒƒƒlmmѧO8;;ÃÉɉûÉF"2 •——så[þÈÌÌlöØ’'­lmm¹rÎFÁ==ùèȉÌP IDATˆ#¬ QTT¤p!¦  @áÎ=yÑÞÞž+ß}ûöEß¾}áèèÈγ;P—LŠÅbdee!==\077¹¹¹(((àDººº°··‡`eekkknĆ¼ÓгgÏFóY1šI&“q'ß\¥&%š››ËÍÝ$OkiiÁÊÊŠKÚÙÙÁÑÑÎÎÎpvvfÓ*R©¹¹¹ÈÌÌDzz:w¬É¯pæåå)ܪdiiÉ%ììì¸ ò:J^O±‹“T*U¡!Å.¿%¿Bž““£0l÷îÝakkË]ì°³³ƒ““wÒeaaÁã·bÚ¢¬¬ ;vìÀwß}‡¢¢"̘1sçÎÅøñãy›/(>>GŽÁ?ü€ŠŠ ¼ôÒKx÷ÝwáîîÎK233ý={öìÉ%úöí‹ÀÍÍ  `#’:©ºº:ܽ{—{ÈËyzz:JJJ<ž:KžhêÓ§×籶¶f£SÕ„L&Caa¡ÂÅùOyý-ð  áàà PÖÝÜÜàîîŽÞ½{³:»uÚa]]233‘––†ôôt…GÆÅÊÊ ŽŽŽ\eáèèÈ%xìííYGi¡  €KàäååqÉfù惘899±NLR[[«pâ"ÑžžŽœœ.élnnØÛÛ+ÔIòQ öööìŠ&Ój>äê&ù‰Wè999Üm!&&&FiÈÚÛÛ³‹aj ¦¦ß~û-¶lÙ"Â믿ŽeË–©ÕÊõµµ8rä¶oߎ¤¤$¼òÊ+Ø´iS»ÞöÈtM999HNNFbb"—LNNFUUàìì¬p2Ù·o_6⬓H$ÈÎÎVÚ—JKKC]]´µµáää„AƒÁÕÕ„»»;úõëÇ[Ðb±©©©HNNFBBîÞ½‹ÄÄDddd@*•BOO...JË9˜Ñ94úä9Snn.ˆFFF\wuuÅ Aƒàææ¾Ã×XŸ ”H$¸wï™™™Éd°³³Sè(4LÐt¶¹ûõRYYÙ(A-üý÷ß>trr‚‡‡ÜÜÜ0hÐ ®Ã7ÍUUU…””$&&ršääddggs·…ÚÚÚ6êØȪ¼%až$“É——§Ðkx‹ŽH$ðøvuWWW®&ï”õéÓB¡çoÑ5œ9sK—.Å£G°bÅ ¼ûî»j=%€L&Ãþólܸ¹¹¹xï½÷°~ýzv’δJNNbccƒ˜˜ÄÅÅqõ‘££#WÉG“¸¹¹5Z¥—é:êëë‘™™©PJNNFjj*Äb1ttt0pà@x{{ÃÛÛÆ ÃÀÙªì<“H$HLLäÊxLL !‘H ««‹4êwôíÛ—õ;º°ÊÊJ®Œß½{—+ó999/Î:lØ0xyyqe]Uó6v6• ÌÏÏGll,€¤¤$®Â—Wnnnððð€««+—d£luT]]ŒŒ ¤§§s[sÇ´¼CceeÅwèLR©)))¸}û6¸Ñ ÙÙÙÉd044„««+wââêêÊÈbu£)JJJ¸ú*))‰ë”eeeA*•B__®®®ÜIú AƒàååÅV¶mG¥¥¥X±bŽ9‚9sæ`ëÖ­µëëëñí·ßbÆ pttÄÞ½{1|øp¾ÃbÔˆH$Btt4®_¿ŽØØXÄÆÆ¢¸¸:::ðððàNöäTÙ€LkI$¤§§#11qqqˆ‹‹Cll,ÊÊÊ ¯¯!C†`Ø°aðöö†ŸŸ_“ 51í#33ÑÑш‰‰All,nß¾ÚÚZ.©ãíí¡C‡ÂÝÝ...,Ë´Zyy9îÞ½‹;wîp—äÉfKKK 6 Æ ƒüüüبr%Ô6AXVV¦pÅ0&&÷ïßg£­˜NïÉQ±IIIHHHàFÅÚÛÛsWBä?ÕyôHg"“ÉššŠØØX®ƒyëÖ-TUUÁÀÀ€K4ÍàèèÈnÉd:­šš¤¤¤p·ø%%%!)) YYYÉd°±±——¼¼¼¸“{v‘£íbcc1sæLvî܉)S¦ðÒSËÎÎÆo¼Ë—/ã‹/¾ÀÊ•+ù‰áÉÇ…ˆˆDDD >>0`À…~ÎàÁƒÙT,L»#"¤§§sIª˜˜ܾ}•••°··G`` ggg¾ÃÕhiiiˆˆˆ@dd$ÂÃÑ——### :Ta”—³³3›Ú‹iwµµµˆç.<ÅÄÄ %%0dÈøûû#00~~~077ç9Zþ©M‚0)) W®\Áµk׃ôôtœ!C‡e·3]Ree%nÞ¼ÉUl±±±HOO‡@ €‹‹ ¼½½1räHÂÍÍïp;…’’DGG#22±±±¸uë*++ahhÈ]m–'? Ànu`˜ÿª¬¬ÄíÛ·¹QqqqHMM…L&ƒ¼¼¼àãダ€x{{³ÛM›ñÃ?`Ù²e7nöïß=zðÒ3#"|ñÅX¿~=fÍš…}ûö±PP[[‹+W®à÷ßGxx8¡¥¥oooøûûÃßß~~~ld éTŠ›7o"22ˆŠŠBYYlllˆ±cÇ"88˜­ÎÞ‚‚‚œ>}—/_FDD>>3f ‚ƒƒ1lØ°.Ù7ø裰iÓ&|ÿý÷X´hßá¨ÌÝ»w1nÜ8ôéÓ.\`õm'››‹Ã‡ãäÉ“ˆ‰‰A·nÝ0a„„„`òäÉ°°°à;D†y*b±ááá ÃéÓ§‘Þ½{#$$sæ̯¯/ß!v"Ÿþ‰£G",, ¹¹¹èÓ§BBBvw£±Š‹‹¹‹[çÏŸGuu5|||0cÆ „††vêU’Uš ÌÊʉ'pêÔ)DEEA__Ï=÷ÆŒƒ1cÆ`àÀ]²ÓÏ0E&“!!!»êùÇ@,ÃÏÏÓ¦MÃôéӻ܊Nÿý7NŸ>°°0\ºt õõõðóóãF[úøø°ÑK ÓÂÂBnN² .àÞ½{°±±App0BBB0vìXð¦Ê>|óçÏÇ¡C‡Êw8*—••…ÀÀ@ 2Çg#´5Pee%Ž;†Ã‡ãòåËèÙ³'^xá„„„ 00% ˜NéÎ;8sæ Ž;†›7o¢oß¾˜7oæÍ›×iYYY8tèöïߌŒ xyyaæÌ™ ÁÀùaÚ]]]"""pêÔ)üòË/())APPæÏŸiÓ¦uºéïÚ=A˜ššŠÃ‡ãÔ©Sˆ‡¥¥%BBB0mÚ4Œ7®KtìF]ÕÔÔàÂ… 8uêÂÂÂP\\Œ!C†`Ú´ixå•W:í$Ì©©©ÜΛ7oÂÌÌ 'NÄÔ©S1qâD¶È è©{÷îáÔ©S8}ú4¢££¡§§‡qãÆaÚ´i˜9s&LMMù±ÝEFFâ¹çžÃöíÛ±téR¾Ãé0 ðóóÃo¼Ï>ûŒïp˜VºvívíÚ…cÇŽA"‘`êÔ©˜?>&L˜À½L—’€àÈ‘#(((€ŸŸ-Z„—^zIãÒ‹Å8zô(öîÝ‹¨¨(X[[cîܹ˜;w.ÜÝÝùa:ŒD"Á… °ÿ~œ:u :::˜5k–.] ¾Ãkí’ ¬©©Á¯¿þŠÝ»w#** ŽŽŽ˜9s&¦OŸ___6ßè!©TŠ«W¯âäÉ“øå—_——,^¼3fÌÐød¾|4Ã?þˆ¨¨(ØÛÛcÖ¬Y†ŸŸ´µµù‘a˜6xøð!Ξ=‹°°0„……BCC±`ÁøùùuŠ;***0hÐ L™2;vìà;œwùòeL˜0çÏŸGPPßá0M "œ;w[¶lAdd$†Ž… "44´S&í¦-¤R)Ο?àøñã°²²ÂêÕ«±xñb›o³¢¢»wïƶmÛPTT„Y³fáÕW_ŸqãئË{øð!þóŸÿà‡~@LL ±víZL˜0A£û¤Ï” ¼sçvïÞC‡¡¦¦S§NÅ’%KÄ’‚ £Ad2þøãìÙ³'Ož„‘‘æÌ™ƒ×_]ãnˆŠŠÂ?ü€cÇŽA*•bÆŒ˜?>«—¦‰Døù矱oß>\»v»­kÁ‚°³³ã;¼§öÚk¯!&&111]ö–Ì7âàÁƒHHH`«Ùª™L†£GbË–-HHHÀ¤I“°víZøûûó襼¼<|õÕWؽ{7ôôôðæ›obÅŠj¿}II ¾þúkìر‰‹/ÆêÕ«akkËwh £–ÂÃñeËœ?ƒÆš5kª™çžô¢¢¢hÒ¤I$hÀ€ôÕW_QQQÑÓ|Ã0j¦°°¾üòK0` š}šÜÝÝIKK‹fÎœIׯ_ç;¤vÃÊ´ú*--¥'NÐ'Ÿ|Âw(̉ÅbÚ¿?¹»»“@  Ù³gSFFßaµZPP-Y²„ï0ÔÂùóçÉÀÀ€|ÈwH­Öªauu5½õÖ[\&ôòå˪Ž«]ù„Ö­[G#GŽ$WWWºsçN‹ÿ¯aœòk¨ÿþª¹YOû}ZKë+™LF¿þú+õë×tuuiÆ T[[ËwXͺ|ù2éêêÒýû÷U¾­¶þM諯¾êðcÁÏÏÞ|óÍÙ£\tt4ÙØØ­­-:uŠïpQÅ1©íïÓbçÿ£îmWff&3†´µµéóÏ?'™LÆ[,2™Œ>þøcÒÖÖ¦qãÆQVVo±4…•uE¬¬ÿº—õ'Nµµ5ÙÙÙÑÕ«Wù§UZLÆÇÇ“‹‹ ™™™ÑîÝ»I*•vD\í¢¢¢‚о}û¸çÔ±Ò{Ö˜$ YYYQkîWÇï¯Z»_Úcÿ);.5T*¥]»v‘©©)õïß¿]“mõý÷ß“¡¡! 4Hc*Û†Zª›$ Y[[·ªL·eǵŸŸ½óÎ;¹æ|þùçÔ«W/’ÉdôèÑ#š4iEFF¶øÿžŒ³¾¾^-„Oû}ZKSë+¢Çåè_ÿúµ{â´½…††ÒÌ™3;d[mù›ž={–æÍ›Gõõõ~,>|˜LLL¨ªªªC¶Ç(Ú¿?éèèÐŒ3Ôväƒ*ŽÉŽnÛ[[Î šó4ýÝö:Çè*}m™LFÛ·o'š={6/·×ÕÕÑ‹/¾H:::ôõ×_óš¨l+ë±²þ˜&”õ‡ÒÔ©SIOO<Èw8-jöˆ:yò$Ѹqã:䪶*ØØØp „¼¼<òóóã9"EíSÿþý[¬ Ôñû«ƒÖî—öÜ KMsÿþ} "###:}út‡n»¾¾ž–,YBZZZôþûï«åü-­ÕRÝÔš2Ý:ª^P—ú§OŸ>í–ÔS‡a{~Ÿ¦hr}Eô¸HFFFjy…Y$‘¾¾>ýöÛo¶ÍÖüMãããÉÙÙ™ÊËËÛôÿÚKUUÓ‘#G:d{Ìÿüð䥥Eÿüç?Õ6a §Šc²£Ú_UyÖøŸ¦½n¯6¾+öµ###ÉÜÜœf̘Aõõõ¶Ýúúzš6mõèу¢££;l»O‹•õÆXYLʺL&£÷ߟ„B¡Z'3‰ˆš\Våüùóxá…°páBœ;wNcW-4h „ââbL™2EEE|‡ÄéȘÔñû«ƒÖî—öÞòãRÙÚÚâüùó˜?>f̘K—.uÈv‰‹-ÂÑ£G†M›6iô*ŸêP7uÔ¶Õ©þÉÉÉá;„vÕßG“ë+pttÄÅ‹±hÑ"„††âäÉ“|‡¤àüùó€I“&uØ6[ú›J¥R¼úê«X¸p¡ÂJÂy,b„ øí·ß:d{ÌcÑÑÑxã7°uëVlܸ€ïš¥éõ“ºyšöº½Úø®Ú×öóóÃ¥K—‰üã¶Ýµk×âêÕ«¸téFÕaÛ}Zšò÷Ô¬¬w,@€M›6aË–-X²d ®]»ÆwHMS–5|ðàuïÞÞzë­ÎW¶¿ßÿˆˆ>üðC@¦¦¦ôúë¯s¯———Ó¦M›hÑ¢EäççG¾¾¾Ü"•••ôË/¿ÐüùóiÔ¨QtðàA233£>}úÐ_ýEááá4|øpÒÖÖ&777ºuë=Î߸qƒÖ­[GNNN”””D#GŽ$mmmruuåF]=MLD‡Ò®ZµŠæÍ›GkÖ¬¡•+W’­­m³WšÚVaa!½ùæ›´jÕ*zçwÈ××—–,YÒâÄà•••ôÁÐܹsiÕªUäããC~ø!w嫹ϕÉdtúôizóÍ7ÉÁÁrrrhܸq$ ¹ {[óžÖî¯æbmj¿¨zÿÉKM·råJêÞ½;åçç«|[ò[Õ}EåÖj©n’_LJJ"ooo …äîîN111Ü{š:ö[[~”m»¾¾ž~þùgzõÕW®ìµTæ)$$„6nÜH‹-"///îŠt[¶ó¬uGSÂÂÂèõ×_Wˆãõ×_§ŠŠŠfco*N"Å„?ýôéééqõpyy9íÞ½›{®¾¾ž¢¢¢hÍš5äääDäááAÝ»w§¿ÿþ»Åz¬-ߧ¹}ØRÊt–úŠˆhýúõdddDééé|‡ÂY½z5ùøøtè6[ú›þòË/€ÚôÿÚÛÖ­[ÉÁÁ¡C·Ù•‰Åbêׯ­X±‚ïPZíiŽÉ–Ú³Ö´¿Mµmi§äsÈ.X°€–/_N†††€{©æœ€ˆèöíÛH›7o¦uëÖ‘––•——7Ù'ikߚ؟ÔÕûÚ/^$…ãLU®_¿NÚÚÚ³®+묬7MÓÊú²eËhÀ€$‘HøE)¥GThh(;Vío+h«†'sD ëäÉ“’¡¡¡dnnN="©TJùùù€ÌÍÍéÊ•+”ŸŸO:::dkkKÛ·o§ÚÚZºwïikk“¿¿?=>¡¼xñ"™˜˜zçwèöíÛtüøq233#¡PÈUm©®®Ž|}}é7Þà^ÏÊÊ"]]Ý+ˆ'·UTTDŽŽŽ «p–••‘««+ÙÚÚ6y²XQQAžžž´hÑ"îÙµk £G¶ø¹÷ïߧ’’êÞ½; ?þ˜òóó)<<œyzz’L&kñ=­Ù_-Ūl¿¨zÿu&2™Œ饗^RévòòòÈÐÐ~þùg•n‡/ÊŽAy§eÆ TXXHááြ¼¼ˆ¨ùcÿáÇ­*?Mm»¼¼\áùÖ”#{{{rqq!¢ÇÇ… õíÛ·MÛiºãiöuK±?gSŸÕ¯_¿Fõ°ü¹ºº:Š‹‹ãÚ…íÛ·Sxx8½ð T\\Ül=Ö–ïÓÒ>ÌÊÊj2ŽÒÒÒ–v_§ðüóÏSPPßapF­v‹q̘1ƒ´µµyï¼Ê뽎¸Å}ýõ×äàà@555|‡¢2­iÏZj‰šn7ZÛ%"úôÓOIWW—[Di÷îÝ€^yå"Rí9“““Bò}ñâÅTXXHDO×N¶õ|¦)]½¯ýÿ÷¨òíøùùÑòåËU¾>±²þ+ëꧦ¦†ìííéÛo¿å;¥Q999¤¥¥ÕiFé4ôäxöìY…ì}Ãǯ¿þJD Æ“ÿ¯oß¾ £““(<'?1lØÁþî»ïÍ;÷©búæ›o%''+lËÅÅ¥Í ÂÕ«W*))QxßÑ£G -[¶Léç|ðÁ€233¹çjjjèÛo¿¥¢¢¢V®²“i'''Ü¿[zOKû«¥X•í—¦´×þël¢¢¢H(Rnn®Ê¶±fÍ:t¨Ê>ŸoÍ%. Õ»wo …DÔºú«5eLÙ¶Ÿ¬÷ZSŽ¶lÙBÿú׿ˆèqgái¶ÓžuGS”ÅÑRìÊÚeŸ¥l.˜'Ÿ“ÇÞp%ðÖü-[û}ÚºÕaEòŽvïÞ=tóæM¾C!¢ÇÇî–-[øC­­-ÙÙÙñ¥§§€QÃyxxÐúõëùC¥ZÓžµÔþµÜn´¦’¯@*?Oxøð! ‘jÏ ÌÌÌ}÷Ýw$•J)99™D"=];ù4çXÊtõ¾ö­[·¥¦¦ªl)))€âããU¶ uÀÊúc¬¬«§uëÖÑ!CøC)m<áòåË000ÀÈ‘#Ÿ|©Ó¹ví† ‚[·n5ùes¯hk7ÚmÐÑÑAMMÒÿÛðý!!!X¶lâããŸ*¦   €³³³ÂóZZMN'Ù¤ˆˆ€©©©ÂóÏC£Ì¹sçvvvÜsúúúø¿ÿû¿6}®²}«££"âþÝÒ{ZÚ_[¶li6Ögñ´û¯³=z4ôõõqùòeÌ›7O%Û8þ<¦L™¢’ÏVw ˶¾¾>¤R)€§¯¿ž,c­ù-•yX³f ÊÊÊ°}ûvhii¡®®®ÍÛiϺ£-ZŠ½=çà’–÷\kþ–­ÕÖ}Ø0Ž®ÂÅÅ}ûöÅï¿ÿOOO¾ÃAii)ºwïÎw àääÄwèÑ£àáÇGåÝ»w h¼š2£ÁÁÁÜÙšhõêÕdccC»víjò=­iÏZj‰Zn7ZÓN­[·Ž\]]éÀtîÜ9úóÏ?)11‘[È@•ç–––ôðáCîßöööÜ¢ ÊÚ붶ñOÛ®uõ¾öÉ“'IOO¯Ù}ô¬=zDººº¦²m¨+묬kº—_~™¦NÊwJ)ÕòóÏ?'+++nÏÎâßÿþ7)¬Ú”ššJãÇ'}}}255¥¹sçRAA=^MgË–-€Œ)**Š"##ÉÀÀ€Ð'Ÿ|B¥¥¥´ÿ~n¡ï¿ÿž+¤òá7ß|C"‘ˆòòòèÃ?TX姭1;wŽ¼½½IGG‡zôèAï¼ó=š^ýuºtéR“'|ʶUTTDK—.%___z÷Ýwiùòå´fÍ®âlJbb"“‘‘Ћ/¾¨°yKŸ{øðanŸíرƒD"8p€„B! Ï>ûŒöîÝÛâ{jjjZÜ_-Ūl¿¨zÿu< +++Ú¶m›J·S[[K®®®´xñb•n‡/ -©TJ»wï&mmm@›7o¦ŠŠ Ú½{7wìòÉ'T[[Ûì±ßš2VSSÓ踮¬¬¤mÛ¶ÒÕÕ¥ƒRyyy‹åè»ï¾####rrr¢sçÎч~HÚÚÚ4jÔ(*((hõvÚ£îP¶gjj*mÚ´‰P(¤;wÒÝ»w[Œ=##£Qœ‰‰‰´yófî¹ýû÷Ó£G(--FŽIB¡H7nÜ €€Z¼x18p€>ûì3ÒÑÑ!ôþûïSRR’B|ÍÕcmù>ÍíCù~o*ŽÎ®¤¤„éÿøß¡pBBB¸• ÕÅùóç ]½z•×8Ž?NÚÚÚTUUÅk]EDDéëëSzz:ß¡<•ÐÐP@ݺukö}Mµgmi›k7¾úê«VµS'NœàN¾>ŒiïÞ½D¤ºsüw€Ï>ûŒÞ{ï=š4iedd‘òþn[ÛøÖÄ®LWîk×××ÓÈ‘#iåÊ•*ßÖòåËiôèÑ{‘•uVÖ5YJJ éëëStt4ß¡(% jûì3ÞâX·n.\¸€¸¸8ÞbèjBCC‘ššŠ«W¯jäœOùùù˜8qb“‹ª "·ß~ "Š+¸çjjjpþüyÌ;·]æg4Ǻuë°wï^¤¦¦ªüÜ»´´ýû÷ÇÒ¥KñÑG©t[ªÂÊ:£‰ª««áëë www9r„ïp”RºôP(ĉ' ‘H0bĤ¦¦vt\ è±»wïÂ××2™ ÇWyrx<—ÈÑ£GñÅ_`Þ¼y¨­­Uù6†éîÞ½‹#F@$áÌ™3j“¯ŸžžŽ‚‚¾Cáüøã8sæ ¯hGGGcÔ¨Q¼m¿+Ú¹s'*++1eÊ[¦¶¶ï½÷vïÞÍw(-zÿý÷±bÅ ¼öÚkÜs†††ðññAïÞ½yŒŽéh[¶lÁ_|C‡uÈÀœ=zàСCøôÓOñÅ_¨|{í•uFUTT`Ò¤I¨®®ÆÎ;ù§IM®Ý³gODDDÀÒÒ>>>سg ×FòÉé[3I=Ãh"®]»0bÄXYYáÊ•+èÙ³g‡mÿùçŸÇÙ³gqæÌx{{ãƶm†a4L&ÃŽ;àããƒ^½z!::666|‡¥`ôèÑ°··ÇÑ£GùEöïßU«VA"‘tøösrrpõêUÌž=»Ã·Ý•uïÞgÏžEFF5jÑ’ÔÔTlÞ¼>>>|‡Ò¢ððpÀ¶mÛ¸òEDˆÇ[o½…ÇóÓQ$ V®\‰õë×ã‡~Àøñã;lÛ'NÄž={°nÝ:¬^½Z£ÎWYYg4MNN‘sçΩ÷Jâ-݃\WWGï¾û. …B7n\—š«èiUVVÒ§Ÿ~JZZZ€V¯^M±±±|‡Å0ÏäÖ­[4vìXÒÖÖ¦µk×’X,æ-–¬¬, "¡PHË–-S›Å†Q×®]£áÇ“ŽŽmØ°×:«%k×®¥¡C‡ò†R©©©ôå—_vøv7oÞ¬°@Ó±²²²hðàÁÔ³gOúí·ßø§ÓÉËˣŋ“ƒƒ™ššÒàÁƒiòäÉôÙgŸ©t F}¤¥¥Ñ¨Q£ÈØؘ×2vìØ1222¢Ñ£GssÓ1퇕uæرcdaaAžžž”““Ãw8-j1A(÷×_‘§§'iiiÑÂ… )33S•q1 £&222hÁ‚¤¥¥EÞÞÞtýúu¾C"""™LF?þø#Y[[“‘‘}ðÁÜêà Ãt]·nÝ¢éÓ§“@  ÀÀ@Šç;¤Ý¿ŸôõõéÒ¥K|‡¢ª««ÉÚÚš¾ÿþ{¾CéÒªªªháÂ…€æÌ™£°0Ã0O§®®Ž>ÿüs244¤!C†Prr2ß!Qbb"yxxP·nÝhëÖ­TWWÇwH £ñrssiÖ¬Y$hñâÅT]]ÍwH­Òä-ÆO>|8bccqðàA\¹rýúõüyópûömÕ od†7·nÝÂܹsÑ¿DFFâСC¸~ýºÚ ç˜?>ÒÒÒ°nÝ:|óÍ7pppÀ[o½…ŒŒ ¾Ãc¦Éd2\¼x“&M‚§§'²³³†+W®ÀÃÃïðZdkk‹¥K—bÆ l:;v쀡¡!,XÀw(]š¡¡!öîÝ‹³gÏ"::ýúõÃæÍ›Q^^Îwh £qˆÇ‡»»;>øà¼÷Þ{¸qã\]]ù îî‰Á;＃7ÂÓÓ§NbíÃ<…òòrlÞ¼nnn¸}û6~ÿýwìÚµ |‡Ö:O“U”H$tðàArww'äååEßÿ=&Ë0®¬¬ŒvîÜI^^^€H‡"‰DÂwh-ª¬¬¤mÛ¶QïÞ½I ¿¿?ýøãTQQÁwh èHzz:mܸ‘H PPP;wŽï°žJQQ™››Ó®]»ø…WYYYdllLGŽá;¦êêjÚ¼y3™˜˜™™­[·Ž ø‹aÔ^]]ýøãäææFZZZ4{ölµ¾Í0++‹fÎœI€ÜÝÝiÿþýj=Eè‹üü|Z»v-™˜˜‰‰ mÞ¼™jkkù«ÍDÏviàêիؽ{7Ž;˜5k–,Y‚#F@ ´C “aU""DGGã‡~ÀÏ?ÿ ¡Pˆ^x‹/†¯¯/ßáµ™L&ÃåË—±oß>n…åY³faÁ‚ðóócõÃh¸ªª*?~{öìATT¬­­±páBÌŸ?}ûöå;¼g²ÿ~,_¾ñññèÓ§ßát8©TŠqãÆÁØاNâ;F‰²²2ìܹÿú׿ ‰0þ|¬^½...|‡Æ0j¥¢¢{öìÁ¶mÛPXXˆ9sæàÝwß…››ß¡µJRR>ÿüsüôÓO°¶¶ÆêÕ«±hÑ"ñè•´´4lݺû÷¹9V­Z…¥K—ÂÄÄ„ïОÊ3'åD"~þùgìÚµ qqq°µµÅ´iÓ0}út@WW·=6Ã0L;‹Å¸rå Nž<‰“'OâÁƒðññÁ¢E‹0{ölób»(++áC‡°oß>ÄÅÅÁÚÚÁÁÁ ÁرcahhÈwˆ ôÂýû÷qúôiœ:u W®\aêÔ©˜?>&L˜¡PÈwˆífúôéÈÌÌDTT”z¯r§ï½÷öîÝ‹„„XYYñÓŒÚÚZìÛ·[·nEff&üüü°`ÁÌœ9³Óô!¦­ä©÷ïß'N/^Œ·Þz öööÜéïÂØ·o^{í5|ûí·xã7ø‡yJ"‘¿üò <ˆ¨¨(XXX 88ÁÁÁ?~<»5‘é4d2®_¿Î]ÄJLLäfóæÍÓøé.Z’––†ƒbÿþýÈÍÍ…‡‡w‡ŽO§I”0LEE.\¸€°°0œ={%%%܈ù3ftÊo*O6TYY‰+W®àÊ•+G||<„B!|||¸„áðáÃÙ­ ó***píÚ5.!™L†!C†`̘1\ctEb±˜Ûo‘‘‘ø믿P]] ggg.Y8jÔ¨NßÙc˜Ž$‘HpëÖ-DFF"22QQQ(++ƒµµ5àïï1cÆ`À€|‡Ê›ØØXL˜0ÇDZcÇ:í”ß|ó V­Z…O?ýkÖ¬á;¦deeq£ö###!ˆ©S§bòäÉpttä;D†iy¢àôéÓ8sæ Š‹‹áî„„À××·ËÍs-“ÉpõêUœ>}aaaHNN†¥¥%‚ƒƒ1eÊva€ÑHYYY8sæ ÂÂÂàÊzgo¿:4Aø¤G!22’K&$$@ `À€ðööæƒîôWϦ-Äb1âããqãÆ ÄÆÆâÆHII 43f üýýannÎs´šE,#&&‘‘‘ˆˆˆÀÕ«WQQQ=zÀËË Ã† ãš:Ÿ Ãt¤úúz$%%!66±±±ˆ‹‹C||<Äb1¸„`@@[èà ?~<¬­­ñ믿vª…Kêêê°zõjüûßÿÆŽ;ØÈÁNL$)ŒÀ(--…““w!Àßß¿ÓŸp1š§¼¼QQQÜÈö›7oBKK Üh¹ÎT'·‡ŒŒ .Y "‚——???6ˆQ;ÙÙÙGDDÂÃÑž={bÒ¤I Áøñã;åHÁ¦ðš |Rii)®]»Æ%}àáá777 4îîîèׯtttø›aÚL"‘àÞ½{HLLDbb"’’’€¬¬,H¥RcèС\2ÐÇLJ]qçIMM âãã¹ÑP±±±¸{÷.¤R)ôõõáêê WWW 8ûɇLg"O&&&âîÝ»ÜϬ¬,ŒŒŒ0tèP…„`¿~ýºÜíWí¥¶¶Ë—/Çž={0wî\lÛ¶M#/Éd2|÷ÝwøÇ?þÞ½{ãøñãlÔhW__›7o*Ì?Z^^###xzzr#ö½½½áììÌê¦]+\øŃ ­­Í%°1zôè.5rH•ÊË˹iEÂÃÃqëÖ-Ô××ÃÆƆäååoooXXXð.Ó îݻǫÅÄÄàÖ­[¨ªª‚‰‰ 7÷|`` †ÊÎÓþKí„O’ÉdÈÊÊBBB’““qçÎ$''#55b±ºººèß¿?\]]áììÌ=\\\`eeÅwø ƒ‚‚¤¥¥!=={Èa‰DÂÃîîî\â[ž`b“þª¯šš¤¤¤ 99IIIÜOy‚Wž8ìß¿?úöí«ð` ¢0êH,#++ ÈÈÈ@ff&îÝ»‡””.hhhWWW¸¹¹ÁÝÝûéèèÈê+8uê–-[‰D‚Í›7cÁ‚³pËÕ«W±zõjܼyk×®Åúõ롯¯ÏwXŒš‘‹CLLŒÂ(dSSSxyyqƒÜÝÝáîîÞiV‰eÚŸX,ƽ{÷””„ÄÄD$''#..999ÐÒÒ‚‹‹ —€–ß ÂÏìUUU¸uë–B’öÞ½{ "8::ÂËË nnn8p ÜÜÜп6ˆiRYY’““¹r.¿“E$ÁÀÀC† Q˜&jÀ€¬ŸÚK6E>úJ>ê*%%…K¾TVVŒŒŒ’†ÎÎÎprr‚££#lmmÙ<‡L»‹ÅÈËËCnn®B0==™ÿÏÞÇEUïÿ ûΰ# (*Š¹›^ܯ[¥æ®i7ÓìÞÊJ-[,KëV¦~53-ÓÌ$Íkj¥i’”¡â)®(ÈŽì ëÌ|~ø›#‹¨0g€×óñ˜‡ÃÌæ58Ÿs>ç}>çskü>êF™uîÜíÛ·ç°¹³pxéÒ¥jß+++i}TõßÖ­[Ã××—>>ÒÏžžžFÛ'ÃÐjµÈÈÈ@rr2RRRœœŒäädܼy)))¸yó&222 kVÕŠÒÑJ:HLLÄÕ«W«g²³³¥å”J%|||¤õ“näíí ´iÓ†#ª&++ )))Òº*55)))HJJBZZRRRP^^055…¯¯oµÑ­º‹ÔÆ'11o¼ñ¶oß???Ì›7³gÏ6ŠÑTZ­ûöíÚ5kpøðatíÚË—/Lj#äŽFÍHJJŠTJHH@||<¤¾½½½½ÞzÌßß_ºïëëËÓÉš˜¬¬,$&&Vë/]»v Pc©sçÎh×®¾7Q•••¸råŠTº³ø žžžh×®]µv777™?Ý µZ›7oJƒ)ªžÅríÚ5¸½]u„©®Íó¬¬×" „uÉÊʪµØ“””„ŒŒ h4·7:îîîhÕª<<|GÅ­[·àììŒÐÐPtéÒÁÁÁRGÜÛۻγÊÊʤÊÍ›7qùòeÄÅTâÿŠ IDATÅáܹs¸~ý:,,,лwo 4<ò øI‰–Z­Fff¦4b?55YYYÈÌÌÔ+leffJÛ‡ªáààGGG½ûÒ@XYYÁÁÁfff°··—¶…ÒöÑÖÖ¶ÑûìjµEEEÒ¶­¢¢*•Jú·¼¼%%%Òv1??(,,¬öo~~¾ôüÛ9¸ººêU«Œ¯:ºKŽí?µ<%%%¸yó¦´mÓí¿W-xéöíKKKõ^kff(•J(•J½v^µí+•JX[[ÃÊÊ 666°´´ÔkßÒ¾»n?½1?oyy¹^û®¨¨ú·EEEP«Õ(,,DYY™Ô¶óòò¤û5µý;999éZu…Uwww½Úhc¤X ”‘¢ZÑ°°°………RÑ«jL·³ygLט”nçüN&&&ptt”FÙYZZJ;ÆwÒh4 ’E×iª©@jkk ©@ª{ÎÁÁÕŠ€,öÝŸüüüjECݺ¨°°°ÚºIW»ó¹ÒÒR½Ñ÷«®Î“££#´Z-ÊËËakk+íÌÔD×Ij¨<ºâ¨n=Tµ8ªT*¥u–î9ݺ¬êsHRcÑjµˆÇŸþ‰sçÎáܹszxÐÑí¤˜™™A«ÕJÅïª,,,ˆ銀½{÷æ÷—Z¤ÊÊJ½bBNNòóó«í@ßY@+,,Dii©t°é^Õ´-ÔõÕU*,--QYYYãv·¶þûÝè •w>«ÞW*•ÒcŽŽŽRaÀÕÕ•#®¨IS©TÒ[·nIíº  @*Š×T<+((¸ï>§®MßÉÊÊ æææR·¦6­+üß;;;X[[ë@ï<Øqg;¯zã)þM „ÍŒntŠnç¸jÁN÷\M;ëZ­¶Æ£¤£‡;vìLž}úÀÑÑ666ðöö†——/®EÔÀî6b¯¦ƒ]5¤{ý¢E‹0yòdôìٳƢœî@€Žn›¦ÛfÖ6¢‘ˆLm#öªŽÜ«:ò¶ê+wþž'NHý]ÝHÄ;Ýyf€n»¶w®¨åbê-""ðõ×_Ëœ„ˆ¨ºíÛ·#""¢ÆS£‰¨~ ¾þúk<ñÄrG!¢{ÄöKÔü±¿K‰'~S½=÷ÜsrG "ªÕÃ?Œ;wʃ¨IÛ¹s'zõê%w "ºl¿DÍû»Ô˜8‚ˆˆˆˆˆˆˆˆ¨ã5≈ˆˆˆˆˆˆˆZ0‰ˆˆˆˆˆˆˆˆZ0‰ˆˆˆˆˆˆˆˆZ0‰ˆˆˆˆˆˆˆˆZ0‰ˆˆˆˆˆˆˆˆZ0‰ˆˆˆˆˆˆˆˆZ0©ÞV¯^Õ«W˃ˆ¨FÑÑј8q¢Ü1ˆš´‰'"::ZîDtØ~‰š?öw©1±@Hõƒ˜˜¹cÕ(99‘‘‘rÇ jÒ"##‘œœ,w "ºl¿DÍû»Ô˜X $""""""""jÁB!w"""""""""’Gµ`,µ`,µ`,µ`,µ`,µ`,µ`,R½EDD ""BîDD5Ú¾}; …Ü1ˆš4…Bí۷˃ˆîÛ/QóÇþ.5&‰ˆˆˆˆˆˆˆˆZ03¹PÓÑ«W/¹#ÕÊ××&L;Q“6aÂøúúʃˆîÛ/QóÇþ.5&…BÈ‚ˆˆˆˆˆˆˆˆˆäÁSŒ‰ˆˆˆˆˆˆˆˆZ0‰ˆˆˆˆˆˆˆˆZ0‰ˆˆˆˆˆˆˆˆZ0‰ˆˆˆˆˆˆˆˆZ0‰ˆˆˆ¨šŽ;bÎœ9rÇ """"`ˆˆèݸqCîDÎÝÝNNNrÇ ¢v?Û0n÷ˆš¶sº,R½ÅÄÄ &&FîDD5JNNFddd£¿OJJ ¦OŸÞèïC$‡ÈÈH$''~ûí7|ðÁ2'"¢úªÚ~ks?Û0n÷ˆŒG}ú»lçt¿X ¤z[½z5V¯^-w "¢EGGcâĉúÙÙÙ5j²²²õ}ˆä2qâDDGG˃ˆîÃÝÚïýløÝ#2.wëï²Óƒ`îÙ¹sç0pà@,_¾K–,©©)ŠŠŠEEEX¶læÌ™ƒ oß¾8qâ4 ¢££ñÊ+¯ ‰‰‰èÒ¥ \\\°yóf¸¹¹A¡PàwÞ‘ÞgóæÍ°°°ÀW_}…ýû÷cÁ‚hÓ¦ ’““1tèP˜™™!$$±±±ÒkΟ?Gy¯¿þ:æÌ™ƒîÝ»ã÷ߨT*|÷Ýw˜5k~øalÛ¶ NNNð÷÷GLL ¢¢¢Ð»wo˜››#88gÏžÕûܵ}6"2BÀ’%K€ . _¿~077GPPöïß_çº(-- YYYX°`žþy¼ôÒKèÛ·/ž~úi¤§§Ö­[‡¸¸8ddd`îܹÒ{ßíuDM‰V«ÅÎ;1cÆ 0Bœùä!Äí•ÚÏkµÚj+¿€€€¯µµµôs}>ݺ¢²²Rzlݺu€˜6mšÞ2U×E/¼ð‚ nݺ¥÷ûvìØ!ˆyóæ !ªw êû:¢¦¤¦íe}Úɯ¶¾º5îÖ¾ó5ìÉíœO1¦{¶jÕ*˜ššbÞ¼yèÙ³'òòòààà€ãÇ£k×®· Ïz·qãÆ ÀÚÚZïwš››ã¹çžÃpõêUTTTàÒ¥K “–ѽöÎ׉ÿZ!¼üò˘>}:V­Z…µk×¢¼¼\ïùš~‡™™Y¿·´´Tú¹>ŸˆŒƒ®WmÛcÆŒpûtŒªËT]EEEõ~_xx8Ô:¯Óý¾ŽÈ˜Õ´½¬OÛ""ùÕÖW¯ÍÝúÏwb¿˜H~lçÔX ¤{6cÆ œý窱_L$?¶sj357îîî"77Wú¹uëÖ",,L ___abb"^|ñE±wï^±fÍ1tèP‘ŸŸ/„¢}ûö€P«Õ5þîÄÄDabb"Þ~ûíjÏé^«Õj¥Çtóê³··–––⯿þß|ópqqÄåË—EzzºÐh4€ ¬ö{«fºó½êóÙˆÈ8tèÐAFzì›o¾~~~"''GQs»¿uë–mÚ´Ñ›|ùå—_–ÖqBáêê*¥‹Ô÷uDM‰Z­¤ Í…¨_Û""ùÕÖW¢ú6Lˆ»÷Ÿï| ûÅDòc;§Æ`úæ›o¾)Ga’š®—_~{öìJ¥Âþýû!„À—_~ Œ5 —.]Âwß}‡½{÷ÂÎÎëׯ‡••Ö­[‡ÈÈHét<¸¹¹éýn'''$%%aÑ¢E°µµ•ß¾};¶lÙFwwwtèÐ;wîÄ–-[ „€µµ5ºwïWWW9r{÷îÅøñãáííèèhœ:u ƒ –-[ðË/¿ ¢¢ýû÷GRRÖ­[µZ +++#22[·n…F£——Ú´i¥RYëgsqq1èߟˆê¶víZäääÀÍÍ ;vDvv6:„õë×ÃÆƦÖu‘ ¦L™‚ÔÔT|øᇸté~øᘛ›ã‹/¾NÅpttÄÏ?ÿ •J…#FÔûuDM…J¥ÂúõëñóÏ?£¸¸~~~ðóóÃÆkm[wnωH>µõÕœœªmÃ@©TÖÚž4i¼¼¼ô^caaÁ~1‘ÌØΩ1(„h s0‰ˆˆŒ@ÇŽqéÒ¥›b€ˆncÛ""""j¾8!ÕÛêÕ«±zõj¹cÕ(::'N”;Q“6qâD^y›¨‰bû%jþØߥÆÄ!Õ[LL bbbäŽADT£äädDFFB­V€ô/Õ_dd$’““k|Žm‹È¸ÕÕ~‰¨yÐõw‰ „DDÔ,”——®_¿xå•W+g$¢fA¥RaÅŠl[DDDDÍç $""""""""jÁ8‚ˆˆˆˆˆˆˆˆ¨cˆˆˆˆˆˆˆˆ¨cˆˆˆˆˆˆˆˆ¨cˆˆˆˆˆˆˆˆ¨cˆˆˆˆˆˆˆˆ¨cˆˆˆˆˆˆˆˆ¨cˆˆˆˆˆˆˆˆ¨cê-""rÇ "ªÑöíÛ¡P(äŽAÔ¤) lß¾]îDtØ~‰š?öw©1±@HDDDDDDDDÔ‚™É€šŽ^½zɈ¨V¾¾¾˜0a‚Ü1ˆš´ &À××WîDtØ~‰š?öw©1)„BîDDDDDDDDD$žbLDDDDDDDDÔ‚±@HDDDDDDDDÔ‚qBªQII ÊËË뵬¥¥%lll9ÑßÔj5ŠŠŠêµ¬B¡€R©läDDMO~~>ê;ÓŒ½½=ÌÌØm$"""j®8!Õè³Ï>ÃÓO?]¯e7lØ€ýë_œˆˆèo·nÝ‚››[½–:t(<ØȉˆšžaÆáСCõZ6;;®®®œˆˆˆˆˆäÂ!Õ(77...õZ6''ÎÎÎœˆˆH_—.]w×å6mÚ„'Ÿ|Ò‰ˆš–Í›7cöìÙw].44çÎ3@""ª¯o¾ù¦Þ#éÇŽ[ïƒjDd<²³³ñý÷ß×kY{{{L™2¥‘QsÇ!ÕjäÈ‘8xð 4MÏ›ššbØ°a8pà€“[·nÅÌ™3¡Õjk]ÆÜÜYYY<بùùùpwwGeee­Ë˜˜˜àË/¿Ä´iÓ ˜ŒˆîfÆŒøꫯ`nn^ë2º¶]VVKKKCE#¢R^^+++¸k[Ÿ>}:¶lÙb¨hÔLñ"%T«©S§Ö97‘S§N5`""¢¿=öØcuv–ÌÌÌ0bĉj¡T*1bĈ:ç477Çc=fÀTDTº‘B•••µÞÌÍÍñä“O²8HÔDYZZâÉ'Ÿ„¹¹ymGRƒ`jõè£Â¢Öç-,,ðè£0Ñßìíí1jÔ¨Z‹†£žˆîbÚ´iµž)`ff†Q£FÁÞÞÞÀ©ˆèn† ''§:—©¬¬DDD„Qcˆˆˆ¨s¤?899aÈ!JDÍ „T+[[ÛZGèèFØÚÚÊŒˆè¶©S§ÖZÜ°²²ÂèÑ£ œˆ¨i=z´túÒ4 Ï 2Rfffˆˆˆ¨ó`¾‹‹ ÂÃà Šˆ\xxx×°°°@DDDgÕ „T§ÚŽXðˆ$ƒ‘#GÖx ÂÜÜãǯµðAD·YYYaüøñ5 ´µµÅÈ‘#eHEDõ1yòdTTTÔøœ……¦N îî5e&&&˜:uj­***0yòd§¢æŠ[ ªÓ°aÃàààPíq 6L†DDD³´´Äĉ«7xƒ¨þj:hnnŽ‰'rî2"#Ö·o_xyyÕø\EEžxâ '"¢ÆðÄOÔz0ÀËË }ûö5p"j®X ¤:YXX`Ò¤Iz;ßæææ˜4iR§4ÊäÉ“«78 QýÕ4—Yee%G$9…BéÓ§×8Ø××={ö”!5´ž={Â××·Úãæææ˜>}: … ©¨9bîêΑ™CDÆdРAzs³XXXà‰'žà\,Dõdff†'žxBïÀŸ‹‹  $c*"ª'žx¢ÆÀ¼HQó2mÚ´ϘáHajH,Ò]õïßÒÏèß¿¿Œ‰ˆˆþfjjª77 çb!ºwUç2ÓÍ]fjj*s*"º›ê=Æ¢QóSÓÁ€ÀÀ@„„„È”ˆš#é®LLL¤«¤é®’Ä ‰È˜T›ÅËË ýúõ“9QÓÒ¯_?i.3Î]FÔ´T=ÍX¡P 88AAA2§"¢†„àà`étbÝéÅD ‰UªÝÎ7wˆÈõèÑ>>>À¹XˆîCQQ&NœhÕªÚµkF#s*"ª)S¦@­V¸=e‹DÍÓôéÓ¥)tÔj5¦L™"s"jn8A¸}*BNNrrr››‹œœTVV¢¨¨HêpèÄÆÆ"66æææ°³³ƒ¹¹9\\\àìì ¸¸¸Ô8Y2Q]òòòPQQ•J•J…ŠŠ äå塲²ÅÅÅÒrºçª*++ƒ³³3RRRPVV†õë×W;=R·¾n°P*•°°°€­­­ôœ“““´n#jN222³gÏââÅ‹HIIAJJ RSSQPP ·lzzº4¯§¥¥%¼½½áåå…Ö­[# ¡¡¡èÒ¥ x2‘ð÷÷ÇC=„S§NA­VcÒ¤IrG"¢F0iÒ$,Z´ðÐCÁßß_æDÔÜ(„BîÔø´Z-7nàƸ~ý:nܸäääj;:¶¶¶Ò¼^º¯ŠndŽnG¾&ŽŽŽðõõEÛ¶máçç'ý???ž¢LÔL•––"==éééÈÊÊBNNòòòj¼åææ"//EEEÕæT©‹¥¥%lllô377‡……²²²àííüüüj¯ËÏÏǽlòlmmaoo'''8;;ÃÉÉ©Æ›‹‹ Zµj…V­ZÁÝÝ2 7nÜÀáÇqøðaDEE!--­ÁßÃÆÆ}ûöÅ Aƒ0hÐ tïÞߢF¤R©påʤ¤¤ ##©©©ÈÏÏGII âââðçŸÂÉÉ &L€¹¹9áááoooid°»»»Üƒˆî"33×®]Czz:RSS‘™™‰‚‚TVV"22yyyèÝ»7BCCacc¥R)µsooo´o߶¶¶r j‚X l†´Z-Ο?èèhœ9sqqq8þ<Š‹‹abb///½Â]ëÖ­áææv×Q€—.]tèÐAïñŠŠ iÔ¡nbVV’““‘””$$ÓÒÒ Õjagg‡àà`„††",, ýû÷GPP‹†DF.;;[jÏÉÉÉR‡%55YYYHMMEaa¡ÞkôŠi5Úìííaaa!è³³³ƒ ,--¡T*ann{{ûzeüá‡0f̘».§ÕjQPP€òòr”””HEÊüü|é±ââbÕZØÔÝÊÊʤßkbbwwwxzzÂËË îîîÒ諶mÛJë^kkë{ûãÕCbb"vî܉o¿ýgÏž­sYÝ΄ T*¡P(““ƒV­Z¡¢¢EEEP©THKK“Ú»V«­õwzxx`üøñ˜4i~øan׉@qq1þøãDGGãܹsø믿pãÆ:Û`}¸¸¸ 44ÁÁÁèÝ»7ÂÃÃáííÝ@©‰è^¥¤¤ ** ÇÇùó眜œú&&&ðóóCçΊþýû£oß¾,Ò]±@Ø hµZÄÆÆâСCRG¢  ®®®èÚµ+ºté‚„„„ 88–––²ä,++Ã… ‡øøxÄÅÅáìÙ³¸uë”J%úõë‡~ýúaÈ!èÞ½;ç#20µZÄÄD$$$àêÕ«ÕFëF ›ššÂËË >>>RÌÃÃCEçéé‰V­ZÁÃÃCš'¥9+))‘Š'ºÑ“™™™HKK“ ¨éééÈÎΖ^ãáá! uEÃöíÛ#((žžž2~jj´Z-öîÝ‹µk×âðáÃ5.Ó©S'ôïß]»vEhh(BBBàààpÏïUYY‰K—.!..çÎÃñãÇSí”ðóóÃüùó1{öl(•Ê{~/¢–(!!»wïƾ}û¤Ó… ! Æ øqãÞ"¶ÝDr©¬¬DTTvíÚ…C‡áÚµky_333ôèÑcÆŒÁc=†N:ä}©ia°‰*..ÆÁƒ±ÿ~ìß¿™™™h×®úõë'!h ^‹/â÷ßGtt4Ž;†ÄÄDxzzbÔ¨Q5j†ÊùÀˆPYYpñâEœ?—/_FBB._¾ŒŠŠ ˜˜˜ÀÇÇG¯x¥»ß¦MøøøpžÑû R©ô ®Uoׯ_Gnn.ÀÉÉ :uBÇŽѱcG¡S§Nðóóã’¨ÕjlÙ²ï¾û.õž³±±ÁÈ‘#ñÈ#`ðàÁÒÕ‰ƒJ¥Btt4~þùgDFF"%%Eïy[[[üë_ÿÂ’%KàêêÚh9ˆšªÌÌLlÞ¼[¶l‘ÎÖ¹“¥¥%:uê„   ¡uëÖðòò‚——”J%¬­­aaaôôt´iÓÅÅÅÐh4ÈËË“X¥§§ãâÅ‹¸páâãã‘——Wã{999áñÇÇܹsÑ­[·ÆüèD-Ê©S§°aÃìÚµ«ÖöçììŒÎ;#88:t€———tÐÝÉÉ ¦¦¦°³³CRR’4Ú¿´´yyyHOOGZZnÞ¼‰ .H·šâ·NŸ>O>ù$§ „MˆZ­ÆÁƒ±uëVüïÿƒF£Aÿþý1zôhŒ=íÚµ“;bƒ¸rå öíÛ‡}ûöáرc033ÃرcáÇs~#¢{P\\Œ¸¸8ÄÆÆâÌ™38}ú4Ο?µZ KKK¢cÇŽÒŽ‡îgžkxÙÙÙRá6!!.\À¥K—œœ !†nݺI·ÀÀ@®[ ]»váÕW_Õ+&( 6 3gÎĘ1cd9H«Õâ?þÀöíÛ±uëV½‹ ÙÙÙáÅ_ÄK/½ÄSœˆpû¢|ð¾ÿþûj;ð666Ç€0`ÀôèÑ£ÁGõ]½zQQQøí·ßpøða¤¦¦V[¦Gø÷¿ÿÉ“'s[CtÔj5vìØO>ù§Nªö¼··7Œððpôïß¿Á÷ç+++qêÔ)DEEIí½¤¤DoKKKŒ;/¿ü2ÂÂÂôý©éa° HLLĺuë°uëVܺu ááá˜6mÆŽ GGG¹ã5ª‚‚|ÿý÷زe ~ûí7¸¹¹aêÔ©X°`Ú´i#w<"£¢‘{ìØ1DGGãÔ©S¸|ù24 <<<†‡zݺuChh(üüüØáoT*.^¼ˆ3gÎàÌ™3ˆE\\JKKagg‡ÐÐPôêÕ  @¿~ýàææ&wdj$)))˜;w.öïß/=fccƒY³fáÙgŸEÇŽeL§¯  ›7oÆš5kpýúuéq???lØ°C‡•1‘|N:…eË–aß¾}zÎrppÀ#<‚qãÆaøðáÕ.ÆÕ˜„8yò$vïÞÈÈÈj£’;tè€W_}œ[”¨4 ¾þúk,_¾—/_Ö{®]»v?~<ÆoðiµJJJðã?bÏž=øßÿþ‡¢¢"é9…B1cÆàÍ7ßd¡°cЈýñÇXµjvïÞ <óÌ3˜2e |}}åŽ&‹äädlݺ7nDJJ Ƈ^x½{÷–;‘,Ôj5Ξ=‹èèhDEEá÷ßGvv6-± IDATœÑ·o_tïÞ]iÆ È›µZ‹/J#COœ8!ÍWÕ±cGôïß_ºñ`Jó°iÓ&¼ð ҅€,,,ðÔSOáÕW_E«V­dNW»ŠŠ lÚ´ o¿ý6ÒÓÓÜÞ ™9s&V¯^Í)D¨Å¸uë/^ŒM›6éä>¾1´!>ŒO?ý{÷îEee¥ô\÷îݱnÝ:ôèÑCÆ„DÆíĉxæ™gpúôié1sss<ú裘;w. dSÆã믿ƧŸ~ªwa3Ìž=ï½÷\\\dLH²dtŽ?.(ˆ^½z‰;w µZ-w,£QYY)vìØ!zöì)ˆŠ?ÿüSîXD‘šš*6nÜ(Æ/áíí-¦L™"þïÿþOÄÇÇ F#wL’J¥GŽo¾ù¦+öïß/T*•Ü1é©T*1}út@º >\\¹rEîh÷D¥R‰×^{MXXXHŸ£cÇŽ"!!AîhDnÛ¶mÂÕÕU¯‡‡‡‹_ýUîhuJNNÏ<óŒ^»511óçÏçö„è*•JÌ›7O˜˜˜HíÅÒÒRÌŸ?_ܼySîxu:|ø°0`€Þ:ÊÕÕUlß¾]îhd`,‘ .ˆ±cÇ …B!ú÷ï/Ž;&w$£wìØ1ñðà …B!Æ/.]º$w$¢¥V«Åo¿ý&/^,„B¡ŽŽŽbüøñbãÆâÚµkrG$#UYY)þüóOñÎ;ïˆ>}úSSSacc#FŽ)Ö¬YÃïN˜˜(ºté"uÖœœÄÖ­[åŽõ@âââD=¤Ïdgg'víÚ%w,¢FQTT$f̘¡·Ó&Ž=*w´{’œœ,fÎœ) …ô9‚ƒƒE||¼ÜшŒB\\œ ’Ú‡B¡³fÍ2úÂà~ùåѵkW½uÖÌ™3EQQ‘ÜÑÈ@X 4åååâÍ7ß"$$DìÛ·OîHMÎÞ½{Epp°°°°o¿ý¶¨¨¨;щ‰‰ÿþ÷¿E«V­"^zé%%*++åŽGMPNNŽØ±c‡ˆˆˆnnn€èÙ³§øøãEZZšÜñè.\^^^R½[·n"11QîX ¢¬¬LÌŸ?_úl¦¦¦bÆ rÇ"jPqqq"00Púž;::ŠO>ù¤IoÃ;&BCC¥Ïdmm->ÿüs¹cÉjãÆÂÊÊJj¡¡¡"::ZîX÷­²²R¬ZµJ:SI7âŸZevêÔ)"lmmÅÇÌS€Z­~ø¡°±±]ºt§OŸ–;Ñ=IHHK—.íÚµDçÎÅòåËÅõë×åŽFÍŒF£Gÿú׿„³³³055ƒ ›6mùùùrÇkñâââôNG;v¬(--•;Vƒûì³Ï„©©©4ÚbÕªUrG"jGŽJ¥RjÃ}ûöIIIrÇjâÅ_ÔMøÚk¯ ­V+w4"ƒÒjµbñâÅz£_zé¥f3Påúõë¢OŸ>zg14µÑÏtïX ”ц „¥¥¥4hOõj@W®\áááÂÒÒRlÚ´Iî8DuÒh4â‡~C‡ …B´mÛV,^¼XÄÅÅÉZˆŠŠ ±wï^1eÊakk+ìííÅüùó9eƒLnÞ¼)|||¤ù´iÓšõ<Ä;vìæ7S(âÛo¿•;ÑÙ»w¯°´´”Úðܹs›ô¨ÁÚìÞ½[X[[KŸsÎœ9,R‹¡ÕjÅìÙ³¥ï¿Ø³gܱ\ee¥xꩧôæTüá‡äŽEˆB”——‹§Ÿ~Z˜˜˜ˆ·ß~›ÓF ÕjÅo¼! …˜?~³9’CÍGAAøøãE@@€011#FŒ?ýô×$«‚‚ñÉ'ŸˆöíÛ …B!þùÏŠð{i EEE"$$DêˆGDD´ˆ¿ý®]»¤ÑH666"&&FîHD÷åðáÃRqP¡Pˆ·ß~[îHê÷ßÎÎÎÒ:kÑ¢ErG"2ˆ_|QúÞ»¸¸ˆãÇË©Q½õÖ[zÅШ¨(¹#Q#aÐÀÊËËŘ1c„££#«ï°gÏaoo/Æ'ÊËËåŽC$ÊËËÅÿû_áìì,ìííųÏ>Ë‘Zdt4Ø·oŸ4²õ¡‡GŽ‘;V³÷Ì3ÏHðAƒµ¨íÖ{ï½'}ö€€NˆNMNBB‚Þœ]ï½÷žÜ‘ âܹszŸ{Íš5rG"jT}ô‘ÞÜ¢-嬟ªÛiGGGqñâE¹#Q#P!È ***0yòdDGGã—_~Ahh¨Ü‘Z„³gÏbÈ!ÇŽ;`ff&w$j´Z-vìØÅ‹#;;/¼ð.\¥R)w4¢:ÅÅÅá•W^ÁO?ý„þóŸX¹r%·_àСC>|8„ðññÁ™3gàêê*w,ƒš6m¶mÛxúé§ñ駟ʜˆ¨~ÊËËÑ»woœ={ðÊ+¯`ÅŠ2§2œ#GŽ`Ĉ¨¨¨€¥¥%Ž?Ž°°0¹c5¸ØØXôíÛ°±±Áþýû.w,ƒyþùç±jÕ*@·nÝpüøqXXXÈœŠ’‰ÜZ’ùóçã·ß~ÃÁƒ¹se@]»vÅÁƒqäÈÌ›7Oî8Ô]½z}úôÁ´iÓ0dÈ\¹rï¼ó‹ƒÔ$„††âÇÄ‘#G°°0<ÿüó(++“;Z³¡V«±`Á! P(ðù矷¸â ¬Y³>>>€Ï>û gÎœ‘9Qý¼þúëRqpøðáxï½÷dNdXƒ Â'Ÿ|àv±4""ååå2§"jXºïvEEà“O>iQÅAøðÃ1xð`ÀéÓ§ñúë¯Ëœˆ „²mÛ6lݺû÷ïG×®]åŽÓâtëÖ ?þø#¶lÙ‚í۷ˇZÍ›7#,, qqqØ´i¼½½åŽEtψ“'Oâ‹/¾À_|^½záüùórÇj¶nÝŠK—.fÍš…áǘHJ¥Ÿ}ö@¥K—Êœˆèî®]»&¨qwwÇ—_~ …B!s*Û;w.{ì1@BBÖ­['s"¢†µvíZi[=~üxÌ™3GæD†gbb‚­[·ÂÍÍ °jÕ*$&&ÊœŠO16€«W¯¢k×®X¹r%æÏŸ/wœmÍš5X²d âââàçç'wjÆJKK1kÖ,DFFbáÂ…xçw`nn.w,º………ppp;†Q¹qã¦NŠÓ§Ocݺu˜9s¦Ü‘š,!qõêUX[[ãÊ•+-þ ÂÀqôèQ·§ éÒ¥‹¼ˆê0iÒ$ìܹÀíbÿÔ©SeN$Ÿììl´k×………pqqÁµk×àèè(w,¢–——‡€€äååÁÑÑW¯^m‘#ýu¶nÝŠéÓ§¦L™Â8ÍGÀâÅ‹1dÈF+æççÃÕÕßÿ}ËiµZ¼÷Þ{X²d úõ뇠  ÄÇÇ7J¦{UßÏð ,X€ððp¼òÊ+ú>Ô²`øðሊŠÂ¡C‡°råJÙ‹ƒ†jc÷ʘ×K«V­ÂàÁƒ¢hlÿmÛ¶ÅÑ£GñÒK/aöìÙ-j®­†öûï¿ãêÕ«€yóæ¬8Xõ;¥V«±dÉ$''׺Œ!-_¾\º¿e˃¾7ѽHMMÅ®]»=ô"""dN$/777©““ƒ¯¿þZæDD cÛ¶mÈËËp{ßÞú†rŠˆˆ@·nÝ‘‘‘ÈÈÈ95Ù©S§ð¿ÿý+W®l´÷033CNN ë\îÃ?Ä'Ÿ|‚åË—cÿþýhÛ¶-òóó-×½¨ïgh|ðöìÙù¨QTTT`ܸqHIIÁ±cÇ0hÐ ¹#0l»Ƽ^zöÙg‘€ÊÊÊF¯7nÔù¼1þÿ™™™á­·Þ¦M›ðÚk¯aýúõrGj’t;Ð … óäVýN™™™áÕW_Å‹/¾(+ï\Æúöí+]à`ûöíÐh4}¢úúꫯ¤ïç²eËZä©ÅwúÏþgggÀ¦M›dNCÔ06oÞ puuÅ¿ÿýo™ÓÈÏÄÄo¾ù&€Ûó(ó`^óÁa#{óÍ71yòdtèСÑÞÃÎÎ^^^ ¬s¹õë×C©TB¡P@©TâÀèß¿£åºõý ¡cÇŽ˜8q¢´R#jHo¿ý6âããqðàA´k×Nî8C¶±{aÌë%333ƒœZœ’’"¦Qcýÿ€™3gbÍš5øÏþÃ/÷HÝ»wúô都÷ß)[[[¬\¹=ö j\Æt§ifff"::ÚàïOTºÓê|||ðÏþSæ4ÆÁÆÆO<ñ€Û1HHH9у¹pá‚t¢ˆˆXYYÉœÈ8Œ1B:ë§7,6¢üü||Xº?~üx“333<ú裀¸¸8dggËœˆ·dhÿþý°´´”.Þ˜žþyØÙÙÕøܾ}û0wî\hµZddd`îܹxꩧðÓO?á•W^A@@Ñ¥K¸¸¸ -- YYYX°`žþy¼ôÒKèÛ·/ž~úi¤§§T*¾ûî;Ìš5 ?ü0¶mÛ'''øûû#&&QQQèÝ»7ÌÍÍ,u¹ßÏÐІ KKK8pÀ ïG-òeËðøãÍiÅwª­i4DGG׺>(**²eË0gÎ 0}ûöʼn'¤×ãå—_FDD/^Œ'Ÿ|ï¼ó:uêTk–¦²^Ò¹pázöì 333tîܧN’ž»[.àvñxàÀX¾|9–,YSSSaݺuˆ‹‹“þ÷óÿg,Þÿ}ܸqCš‹î.&&Fºß»woƒ¿Mß©Q£Fá‹/¾ÀåË—k]ÆBBB`cczíÈX?~\ºß·o_“ŸîÝ»Ks/ÿñÇ2§!z0ºï°……zôè!sãÒ¯_?é>æ5‚Í‚ Ä€äŽ! :tè „¢¼¼\ÄÆÆ @¬ZµJ=zTL˜0A\¼xQ´mÛV¼ûî»ÒkóóóE§N„···HMMF¤§§ ÂÉÉIüúë¯"==]˜›› ooo±jÕ*QVV&._¾,ÌÌÌŒêï Ó¯_?ñÜsÏɃš‰ÌÌLabb"Ž9"w”{V×ú ;;[Œ9R¤§§KËOš4I899‰¼¼>^Z¾õÖ[rÇ¡Àa#7nœ˜4i’Ü1$5í„ ¢¤¤Dzì…^Ä­[·ô–ݱc‡ æÍ›'„B«ÕVûÕŠþþþÂÚÚº¡?Λ8q¢?~¼Ü1¨™øî»ï„©©©¨¨¨;Ê}«i}pàÀiÃçm×®]båÊ•€¸páB¿ënŒ}½¤+j4é±6mÚSSÓ{Ê¥T*±nÝ:¡ÑhÄ… DAAA­ƒ¦ê믿–––z/ªÝ³Ï>+µ§¬¬,¹ã!„ÈÍÍÄСCåŽ"(www¹£U³`Á©ýêøÐßÂÃÃááá!w¢âîî.ˆððp¹£ŒŒ i=È7ÍO1nDÙÙÙðôô”;FtW[³¶¶–‹ŠŠ8::ê-«›I7YxMWj333«ö˜¹¹9JKK$oCjÕªU½æý"ªŒŒ 8;;K§Ô4E5­Ž?Ž®]»BÜ> ¤w7nöîÝ Õæ\|+9ãz©êÜRVVVÒU+ë›kÕªU055żyóгgOäååä(†æåå…òòräääÈ¥IÈÍÍ•î;99ɘäoööö wŠ¼\”J%ý¿‘±¨ú½Ô]µ—þ¦Û.r{@M®­ßÙ×#ÀÅÅEºÏ¶Þ<°@؈ìììPXX(wŒ{¦›`þÆzë:?º9šº‚‚iGˆèAÙÙÙ¡¨¨Hî N£ÑàÊ•+(//¯ñ9Ý…~{]g¬ë¥úæš1cNž<‰Áƒ#66ýúõÃÇlЬ† û>4Çâgc¨ú½U©T2&ùÛƒ÷šîobÌsoRËUµý˘Ä8•””ûÚÔäé¶Aºï4ý­ê¾Os©´t,6"///¤¤¤Èãžé.ªòóÏ?ë=®û,£G6x¦Æ’’///¹cP3Ñ­[7”••áâÅ‹rGiPÁÁÁP©TX·nÞãéééX·nÚ·oàöEGªjè+Ùëz©¾¹V®\‰°°0üòË/ؽ{7 –.]*-/„0PâÆ‹XZZÊ¥I¨:êÈXŽ¼ë ÞÞÞ2'ùûoRu„‘±puu•îߺuKÆ$ÆI×~«þˆš"Ýw˜£Ù««ºîsww—1 5Qpp°Ñ *++@:-NG·_õñE‹! |ðòóó¥Ç?ýôS„……á?ÿùÞk«îØÖôûjZNnB\¼xAAArG¡f"$$;wÆ7ß|#w”ûVSû}ôÑGáëë‹… báÂ…øᇰvíZ̘1Ó§OǼyó/¾ø"víÚ… .`ýúõÈÌ̼ëû5…õ’î5U_«V«¥ßQß\}ôòòòcÇŽ…Üîxfff"--­Î,ÆN«Õ"22“'O–;J“Ѻuké¾±üÿ§¦¦çªÊwÒÚ}||dNBT]@@€tÿúõë2&1>Z­×®]øûûËœ†èÁøùù¸ÝÎiÖ$&&J÷u'jÚX lD£GÆÍ›7qæÌYs\¾|ï½÷€Û+¶O?ý±±±Xµj•tZܲeËpá·Ô?~#GŽÄÈ‘#ñòË/ã¹çžp{¾-[[[dggã¿ÿý/€Û#‰¢££qìØ1©3ÿþûï#77_}õ’’’7n4š#¬gÎœAJJ yä¹£P3òÚk¯aõêÕF³£__*•ªÖõ­­-:„!C†àÿþïÿ0mÚ4œ8q[·n…££#† ‚¯¾ú ööö˜,Vñá‡báÂ…€#GŽ`àÀ2'"ºGŽ‘¦”ùè£ðüóÏËœÈxôêÕ 'Nœ€»»;222Œjcº?AØÈ^yå|úé§ÍòâMUaa!6lØ€—^zIî(ÔÌ( |ñŸté¦OŸ.BKÔœ]¹r#GŽÄàÁƒY¼G2d€ÛóXÊÝWø믿oÐÙ±cÀÒÒC‡•9 Qu …cÇŽ?~œ#‹ªØ¼y3ÀÃÃýû÷—9 уéß¿¿4¿Þ_|!sãñ×_Ig?Œ7ŽÅÁf‚ÂFWWW|ôÑGrG¡ÿï¿ÿý/<==!wj†¼½½±gÏüüóÏxì±Çšä•ÌBEEãš{”ÞÉ“'Ñ¿øûûcÛ¶mrÇi’tÛ¢’’lß¾]¶999X²d 8'''Ùr·çcüñÇÜž®ÅÑÑQÖ1‰ñøᇤéA¦OŸ333™=sssL:Àí3&8 s"ã°råJé>7,62¼ÿþûx÷Ýw+wœïäÉ“X¹r%V®\ ~ý©qôìÙQQQ8sæ ºtéÒ¢N;ºzõ*V¬X!MØþî»ïB¥RÉœŠšV«Åûï¿~ýú¡W¯^ø駟`cc#w¬&éÑG…½½=à½÷Þ“Šë†TYY‰7bË–-z^Ë»ï¾+ý¦M›&s¢ÚõîÝ¡¡¡€-[¶´ø©kÔj5-Z055Åœ9sdNDÔ0æÌ™#í;.Z´¨ÚöZš3gÎH5»uë†îݻ˜ˆ +$0jÔ(Ìœ9&L&ì%ÃËÏÏÇĉ1kÖ,Œ9Rî8ÔÌ…††âܹs Á?þñ¼õÖ[ÒÕo›³víÚaÑ¢EB@W_}UºP5iii6l^ýu¼ûî»Ø³g¬­­åŽÕdÙØØà_ÿú )) 6l0xsss,Z´Hö‘ƒÀí¿ÁÆí۷ǨQ£dNDT·åË—¸}àdþüù-ºp°zõj½Ñƒ2'"j:uÒE¸víZ™ÉG£ÑàÙgŸ…V«ð÷:š^¤Ä@JKKÑ»wo(•Jüøãia`*• #FŒ@QQŽ?+++¹#Q !„Àúõë±páBàý÷ßLj#äŽEtÏJKKñÑGáƒ>€§§'¶oߎnݺÉ«YÈÎÎF@@ŠŠŠàè舸¸8øúúÊËàtz:tè`Û¶mœ„š„ÁƒãÈ‘#€eË–aéÒ¥2'2¼Ó§O£OŸ>¨¨¨€­­-.^¼¹c5˜äädtìØ¥¥¥°´´DLL ºté"w,ƒ{ã7°lÙ2·×}¿üò‹Ì‰¨!q¡X[[ãûï¿Gbb"F’’¹#µ%%%9r$’’’°{÷nÉ  æÍ›‡³gÏ¢}ûö9r$ˆ“'Oʨ^Ôj5>ÿüs`ÅŠxñÅqúôi›››tZ^AAfÏžÝ"çïüôÓO¥â`÷îÝ¥«»Ï?ÿ€·ÞzK*¶ÙÙÙ˜4i’45À‡~Èâ 5;¾¾¾øðÃååå˜4i²³³eNeXF *•J|þùç2'¢†Æ¡ùûûã×_ÅåË—1bĈ·B‘Cff&† ‚ÄÄDüúë¯ðóó“;µPؽ{7þøãTTT W¯^;v,Ž=*w4¢©T*lØ°;wÆüùó1vìX\»v K—.å(øFðòË/£gÏž€_~ùo¼ñ†Ì‰ +&& .XYYaË–-œ+˜š ???¬_¿ÀíÓïÆŒƒ˜˜™SFII ÆŒƒÿ×ÞÇUU'þ_VpAÜDPqiQs©qÅ%Ó‘Ô¦œ,¿eéLfßúŽK5Úf‰iš¦¥•¦i¸´©[˜ë(Š[¸€ˆ ˆ²Èz~ð󎌖Xȹx_ÏÇÃGtÎ}xßáÀyßÏrüøqIÒÃ?¬1cƘœ ¸3ž~úi…‡‡K’Ž9¢~ýúÙÍÀŸØØX 2ĺŒÂ¼yóÔ°aCsC¡Ìñ›W9kÚ´©6oÞ¬äädµoß^{öì1;Ò]kçÎêСƒÒÒÒ­Æ› Ðý÷߯mÛ¶iõêÕJKKÓ< 6mÚè_ÿú—®^½jv<@§NÒ‹/¾(????^]ºtQ\\œæÌ™£:uê˜ï®åää¤Å‹[×ì|íµ×´téR“S•'N”¸Ézã7`r*àö >Ü:øZi¶k×.“SÝY—/_VŸ>}¬eè½÷Þ«Å‹›œ ¸³/^¬Ž;J*.ÍÂÃÃuåÊ“SÝY;wî,1 ròäÉ:t¨É©p'Pš Y³fÚ¹s§¢9sæXùÄWTT¤ÈÈH………)88X±±±”ƒ°9ýû÷Wtt´öìÙ£öíÛëþçäçç§ &0ýå.''G+W®Ô€Ô¤I}öÙgš8q¢Îœ9£?üPM›65;¢]ð÷÷ײeËäèè(Ã0ôøãëË/¿4;ÖuêÔ©³*ž~úi7ÎäTÀï3}út=ýôÓ’Š§Ý>ðÀúæ›oLNug$‚¹> IDAT''«K—.֙͛7WTT#Ìq×sww׺uëÔ¬Y3IÅ£þÃÂÂtîÜ9““ÝëÖ­S·nÝtñâEIÒرc5eÊsCáÎ1`š‚‚cÊ”)†³³³b9rÄìHÞáÇN:...Æ´iÓŒÂÂB³#¥rþüyãŸÿü§`H2š5kfLž<Ùˆ7;îRÆ7ß|cŒ5ʨ^½ºáììlôîÝÛX¾|¹‘ŸŸov<»6wî\Ãb±’ GGGcÑ¢EfGº#âããúõë’ IFÏž=ÜÜ\³cHQQ‘1~üxë×µ³³³ñæ›oEEEfG+3[¶l1üüü¬¯±cǎƹsçÌŽ”«ääd£}ûöÖï???cëÖ­fÇ*3………ÆŒ3 '''ëk|á…îªknDAhöîÝk´k×Ψ\¹²ñòË/—/_6;R…séÒ%ãå—_6*UªdtèÐÁØ¿¿Ù‘€ßmß¾}Æ‹/¾h4hÐÀdÿøÇ?Œ;wRzã¹r励zõjcôèÑF:u ‹Åb„††sçÎ5.\¸`v<\çƒ>0 I†Åb1^y壠 ÀìXeæ»ï¾3jÕªe½éèׯŸ‘““cv, ÌÌœ9³Äõƒ>hœ8qÂìXHvv¶ñÊ+¯”x]}úô1®\¹bv4ÀW®\1zõêeý~prr2þñÙÙÙfGûCŽ;fôìÙ³Ä늌Œ4;Ê¡ÈÏÏ7Þ~ûmÃÃÃèU«–ñÞ{ïñ.z)äææï¾û®Q«V-ÃÓÓÓ˜9s&#_p×(**2¶lÙbŒ?Þð÷÷7$µk×6}ôQãóÏ?7RSSÍŽˆ àðáÃÆÌ™3îÝ»...†‹‹‹Ñ­[7ãí·ß6N:ev<ü†¥K—®®®% †¤¤$³cý!ùùùÆÔ©S GGGëë9r¤‘——gv4 ÌÅÄÄ”%[¹recêÔ©²<ˆŠŠ27n\bdäŒ3M»WXXhLŸ>½DqÞ¤IcݺufG»mÙÙÙÆäÉ“Ê•+[_Kýúõ-[¶˜ å„‚ÐƤ¥¥/¼ð‚Q¹reÃÏÏÏxóÍ7ôôt³cÙœôôtã7Þ0üüüŒÊ•+'Näó„»ÞñãÇÙ³g½{÷6*W®l8::íÚµ3Æg¬X±ÂHNN6;"LVXXh8pÀ˜3gŽ1|øp£^½z†$ÃÛÛÛ=z´ñå—_—.]2;&nÃÏ?ÿ\¢`ððð0æÏŸ_!oÊþùg£M›6% †Y³f™ ¸£.^¼h 0Àúu/Éðõõ5fÏž]!FÍþðÃFHHH‰ü76vìØav4À¦ìر£D‰.ÉèÒ¥‹±iÓ&³£ÝRNNŽ1kÖ,ÃÇǧDþ2 ÁÎX Ã0ÊrMC”3gÎhÖ¬YZ°` ÃÐc=¦±cÇÊßßßìh¦Š×œ9s´hÑ"9;;kôèÑ7nœüüüÌŽ”«ììlEGG+&&F[¶lÑ®]»”››«fÍš)$$D¡¡¡j×®Zµj%'''³ãâÉÈÈÐÞ½{µsçNmݺUÛ¶mSZZšj×®­N:©K—.zàÔ¶m[Y,³ãâwJMMÕˆ#´aÃ뱎;êŸÿü§zôèab²Ò9uê”&Ož¬O>ùĺ)›ŸŸŸ–-[¦“ÓåãË/¿Ôøñ㕘˜h=æë뫱cÇêñÇ—···‰éJÊÍÍÕªU«4gÎmÛ¶ÍzÜÅÅE&LÐ+¯¼bÝqÀdffêµ×^ӻﾫ¼¼<ëñÐÐP;V”«««‰ KJNNÖÂ… 5wî\={Öz¼~ýúš9s¦lb:˜‚ÐÆ]¹rE .ÔìÙ³uâÄ ÝsÏ=1b„†ªÚµk›¯\œ?^_|ñ…/^¬Ý»w«iÓ¦zöÙgõ—¿üEÕªU3;`rrr´k×.ÅÄÄhëÖ­Ú¾}»._¾¬J•*)((HíÚµSûöí¬Ö­[ËÅÅÅìȸM©©©Ú³gO‰?'NœajØ°¡BCC­å°¿¿?…à]Æ0 }üñÇúÛßþ¦ôôtëñîÝ»kÒ¤IêÞ½»Íý›Ÿ8qB³fÍÒüùó­7J‹Ecƌь3äîînrB |effjÆŒš5k–233­Ç®ˆˆ=ôÐCª\¹r¹g3 C»víÒ_|¡%K–Xw—Š¿oû÷ï¯3fØý` 4:¤—^zIQQQº¾n©S§ŽFŽ©GyD:t0åçvvv¶¾ûî;}úé§úú믕ŸŸo=W­Z5=ÿüóš8q¢ÜÜÜÊ=ÌGAXA†¡-[¶è“O>ÑŠ+”­nݺ©_¿~êÓ§6lhvÄ2• uëÖ)**J›7oV•*U4tèPEDD(44Ôæn‚[STT¤cÇŽÝP(]ºtIÎÎÎjÑ¢…Z¶l©-Z( @þþþò÷÷7å¦%]¸pAqqqŠ×‘#GtèÐ!>|XgÎœ‘$5jÔÈZö^+~íå #H)))úÛßþ¦Ï>ûÌ:O’4nÜ8 6ÌÔâ­°°P7nÔœ9s´víÚÛ¶m«Ù³g3jv/55Uï¼óŽæΫŒŒŒçªV­ªÞ½{«W¯^ UÓ¦MïXŽôôtmݺU7nÔW_}¥Ó§O—8ïàà þýûkòäÉjÛ¶íËÜ­öìÙ£iÓ¦)**ªÄÏC©x”ÞÀÕ£G…„„ÈÃÃãŽå8~ü¸bbb´~ýzmØ°AÙÙÙ%λ»»ë™gžÑ„ T£F;–¶‚°ÊÉÉÑúõëµfÍmØ°A/^T`` |ðA………©sçΪY³¦Ù1oËÅ‹µmÛ6ÅÄÄèÛo¿U\\œj×®­^½z©ÿþêÛ·¯M Ç*ª_~ùE{öìÑtøða>|XGU^^ž,‹4h 5oÞ\ 6TÆ Õ¨Q#5jÔˆÑ>eÄ0 %''+!!A¿üò‹Nž<©“'O*>>^ñññJKK“$yzzªeË–Ö·mÛ¶j×®<==M~°û÷ï×Ë/¿¬uëÖ•8îêêª^½zièСêÝ»·ªW¯~dzjÇŽúâ‹/ôå—_êܹs%Î7mÚT¯¾úªyä988Üñ<@E‘™™©Ï?ÿ\óçÏ×®]»nú___uîÜYAAA P«V­äçç§*Uª”úy tîÜ99rDqqq:xð bccuðàÁJ IòööÖã?®¿þõ¯wÝ À Z°`.\¨”””Î;88¨uëÖêر£¨æÍ›ËÛÛû¶– ÊÎÎÖ™3g§¸¸88p@Û¶m+1}øz;vÔ˜1c4lØ0– €$  ¯°°P±±±Z»v­6nܨ={ö¨°°P-[¶TçÎÕ®];)((¨\nJ###CÐÁƒµgÏmݺUñññrttTûöíÕ½{wõíÛW÷Þ{/7@9(,,TBB‚u¤ÚÑ£GuôèQ%$$èìÙ³Ö©žžžÖÒ°Aƒòõõ•———êÖ­«ºuëÊËËKµjÕ2ùÕ˜+??_çÏŸWrr²Î;§ääd%''+))ÉZž:uJ¹¹¹’ŠËœ ¨qãÆjÑ¢…u$g«V­ˆRÙ³gfÏž­Ï>ûÌúuu“““õçj—.]Ô¦M›2Yç,++KÔŽ;´iÓ&EGGëòåË7<.44TÏ<óŒ ÄZ¨À->|X+W®ÔªU«´wïÞ[>ÞÝÝ]¾¾¾rwwWÕªUåêêª*Uª(??_™™™ÊÏÏ×¥K—”’’¢óçÏß´¼ž···  Áƒ+,,LÎÎÎeõÒüùùùÚ¼y³V®\©Õ«Wëüùó¿ùxyyy©N:òðð³³³ÜÜÜäìì¬ììlåææ*++KJJJºaDòÍkРA>>òññ±ŽÄ¼~dfݺuY2eâÂ… Z²d‰–/_®;wþêã¼¼¼¤úõë«^½zòõõ•§§§õ†ÃÃÃCYYYÖr!''G‰‰‰JNNÖéÓ§uøða?~üWË___ 2D#GŽTppðz¹À]-11QÑÑÑŠŽŽÖ¶mÛtäÈ–ésÔ­[W÷ß¿ºtéb}7æòSXX¨ýû÷+&&FÑÑÑŠUrrr™>‡“““š7o®ÐÐP…††*,,LõêÕ+ÓçÀÝ…‚Ð$%%éàÁƒÚ¿¿:¤'NèäÉ“:{ö¬õ|GGGÕ¨QC5kÖTÍš5U£F UªTI...ÖÒîÚ† W®\‘T\FæååéêÕ«JKKSjjªRSS•––fý%ÆÁÁA>>>jØ°¡š4i¢€€µiÓFAAAòññ1á³ ¬¥¥¥éܹsJIIQRR’Ο?oýïÍ µëC¾ž›››\\\äááa½ö\+-þ{Z­‡‡G‰bÍÁÁÁ:º  Àzº&77·Äz+yyyÊÊʲ^Ç®åÊÌÌ´¾#{3U«V½¡ä¬Y³¦|||T§N£*½½½YÓ¦HHHЪU«´iÓ&ÅÄÄ”Ø ¡¬9::*88XݺuSŸ>}BÉ”±«W¯êðáÃ:xð Ž=ª3gÎèܹsÖC×F fffZ~ºººªzõêòòòR½zõäíí­¦M›ªU«V ¬pËö 55UСC‡tìØ1¥¤¤(11Q)))ÊÈÈ°þþš——gýÙÍÍMîîîªW¯ž¼¼¼äçç§-Z¨U«V `™.Ü B;–——§S§NéôéÓºpáB‰‚/55U¹¹¹ºzõªrrr$IG•$5oÞ\’T¹reUªTI®®®ÖRñZÁxíâT¿~}vKPBfff‰â033S999º|ù²òóó•‘‘a-ô®\¹bÉtMQQÑ S(òóóuîÜ9?^Mš4¹a¡gGGÇË,\û…ªJ•*ruu•»»»\\\T­Z5ëµÍÝÝ]nnn%F;r=CESPP ]»viïÞ½Ú¿¿öíÛ§øøøRMCúo®®®jÒ¤‰‚‚‚Ô¶m[µmÛV÷Ýwß]X僂¥!IZºt©ÉIàFË–-SDD„ø±ÜZff¦Îœ9£¤¤$¥§§[G$dddÈÍÍMŽŽŽòôôTåÊ•åëë+yyy™w+FØ777µlÙ’Ê ‰„€]ciÀŽQvŒ‚°c„€£ ì!`Ç(;FAˆR‹ˆˆPDD„Ù1খ-[&‹Åbv ¨p(;ædvT÷Þ{¯ÙàWÕ¯__C† 1;T8Ã0 ³C0SŒ;FAØ1 BÀŽQvŒ‚°c„€£ D©ÅÆÆ*66ÖìpS§OŸÖŠ+ÌŽŽ“ÙPqDFFJ’–.]jr¸ÑÖ­[!Ã0ÌŽ #;ÆB”Ú¸qãÌŽ¿*$$DË—/7;T8ƒ¹X€ÝbŠ1`Ç(;FAØ1 Bü.þþþ=z´Ù1সF@éQâw©S§Ž<==ÍŽ7Å5 J]Œ;ÆB”Zdd¤"##ÍŽ7µuëV=òÈ#fÇ€ ‡‚¥«Ÿ~úIË—/רQ£Ô¥K†¡ŸþYÿû¿ÿ«&MšèСCêܹ³œ uëÖ™€8}ú´V¬XÁ5 n!n‹ÅbQ¯^½´dÉ?^EEEÊÈÈÐœ9sôË/¿hÑ¢Eš;w®–/_®ääd…‡‡k÷îÝfÇ`G¸FÀía BÜ6Ã0äàà -Z(>>^’Ô¢E =zTùùùrrr’$}ðÁ;v¬FŒ¡%K–˜€á·‡„¸m‹åW]»ñ–¤~ýúI’öïß_>Á@\£àvQâŽñöö–$¹ººšœnÄ5 ŠQâŽIOO—$õèÑÃä$p#®QPŒ‚·­°°P’TTTtùëmܸQ5Òßÿþ÷rË\£àöPâ¶deeiöìÙ’¤S§NéÓO?Õ•+W¬ççΫ˗/+11QGŽÑöíÛU£F ³â°3\£àö±‹1Ê„¿¿¿Ž9"¾œØ"®QðëAØ1 B”ZDD„"""nz®   Ä ¼-[¶L‹å¦ç¸FÀ¯£ Ä’••¥3f(!!A’ôâ‹/j÷îÝ&§€b\£àÖXƒ¥)I7nœÉIàF[·nUdd¤–/_nv¨P(;ÆcÀŽQvŒ‚°c„€£ ì!`Ç(Qj±±±Š5;ÜÔéÓ§µbÅ ³c@…ãdvT‘‘‘’¤¥K—šœn´uëVEDDÈ0 ³£@…ÂBÀŽ1‚¥6nÜ8³#À¯ ÑòåËÍŽŽÅ`.`·˜b Ø1 BÀŽ±!njÙ²eš>>òññQÍš5åîî.‹Å"777IRff¦ ÃPFF†RSSuöìY%%%éäÉ“Š‹‹S\\œNž<©›}i999©[·n=z´  ggç;þ9`ÛvíÚ¥ùóçkåÊ•JOO¿écjÔ¨¡ÀÀ@µjÕJ-Z´êÖ­+///yzzÊÑÑQnnn***RAAòòò”““£ôôt%''ëìÙ³:sæŒ:dý“——wÓçjÙ²¥FŽ©Ç\uêÔ¹“/lá]êâÅ‹š5k–,X ”””ç,‹uúoXX˜BBBäééY¦ÏŸžž®-[¶(::Z?þø£öîÝ{Caèíí­Ñ£GëùçŸgT!`g ôùçŸkÖ¬YÚµk× ç}}}Õ½{wuíÚU¡¡¡jÚ´i™>~~¾víÚ¥èèhEGG+&&FÙÙÙ%ãêêªjâĉ .Óç[BAx—¹xñ¢Þzë-Í;W™™™Öã Ó Aƒ4`ÀÕ«W¯\s%&&ꫯ¾ÒªU«£¢¢"ë9777=û쳚0a‚jÕªU®¹”¯ÂÂB-]ºT¯¿þºŽ=Zâ\Ó¦M5xð` ûì3IÅ7Ý#FŒÐ[o½UawÝ„Lbb¢ºwïnÝý300P6l¨PSõJãôéÓêÝ»·âââ$ÒÙ¼y³¼½½MNà·†¡¿þõ¯ú׿þ%IªR¥Š–-[¦ððp““•­‚‚;V ,T¼¦â—_~©¾}ûšœ n[ÅV êÝ»·µìÚµ«¶lÙrוƒ’T¿~}mÙ²E]ºt‘T¼˃>hÖÀ6½ð Ör°fÍšÚ¸qã]WJ’“““>üðCM:URñZªC‡-±™ TŒ ¬ Õ£Gýøã’ŠËÁuëÖ©J•*æ»Ã²³³Õ³gOmß¾]’Ô½{w}ûí·rtt49€ÿöî»ïj„ ’$wwwmÙ²EAAA&§ºóf̘¡—^zIRñëŽU‹-LN¥Ç bêÔ©Ör0((H«W¯¾ëËA©xzâúõë­%ÃÆõú믛œ ÀÛ½{·&Mš$©øûvõêÕvQJÒ¤I“ôüóÏK*é=|øpåå噜 J„ÀO?ý¤ÊÃÃCûöíSƒ ÌŽU®Ô¶m[]¾|YŽŽŽÚ¾}»î¹ç³cPñôÚ6mÚèÈ‘#’¤ hôèÑ&§*_EEEzðÁµqãFIÒ‹/¾¨3f˜œ J‡„6Î0 ?^………’¤yóæÙ]9(I5Ò| ©xºõøñãE· ؆÷ßßZÉ£ üIDAT<ØîÊAIrppÐ'Ÿ|¢ÚµkK’Þ{ï=ýòË/&§€Ò¡ ´q+W®ÔO?ý$I0`€†jr"ó >\ýû÷—$mß¾]kÖ¬19€ôôtë´wwwÍ›7ÏäDæ©[·®ÞyçIÅ£*_yå“@éPÚ¸ÈÈHIÅ;f2]­x3'''IÒ¬Y³LNàÓO?Uzzº$饗^R­ZµLNd®ˆˆµk×N’´bÅ ;wÎäDpk„6ìرcÚºu«$iĈìŠ)©eË–ÖQ”ÑÑÑ:qâ„ɉû¶páBIR­ZµôÜsÏ™œÆ|š2eŠ$©  @‹/67”¡ [¶l™u½'žxÂä4¶c̘1’Š×g\¶l™ÉiûuèÐ!íÛ·ORñȹJ•*™œÈ6ôêÕK¾¾¾’Ä5 @…@AhÃ6mÚ$IòööVçÎMNc;BBBT§NI²î  ü]ÿý7xð`“Ø'''…‡‡K’þýïëÂ… &'€ßFAh£òòòôóÏ?K’:uêdrÛb±X¬…éÏ?ÿ¬‚‚“öiûöí’$uìØÑä4¶åú7u¶mÛfb¸5 BuüøqåääH’Ú´icrÛ$IÊÎÎÖ±cÇLNاÿûß’¤fÍš1½ø¿´nÝÚúñµÏØ* BuþüyëÇ5jÔ01‰mº~§T¦ïæ¸xñ¢$©víÚ&'±=×NRSSML·FAh£222¬SÞ¨fÍšÖÓÓÓMLد´´4I’»»»ÉIlÏõ×( B¶Ž‚ÐFU­ZÕúqvv¶‰IlÓõŸ“ë?WÊ›››$®Q7såÊëÇUªT11 Ü¡º~Ô £OntýìëGê(?צú_Iˆÿ¸6ýZ’u×u°U„6ªaÆrp(þç9~ü¸ÉilÏ/¿ü"IrppPãÆMNاFI’d†Éil˵k”ôŸÏØ* BU£F µlÙR’´cÇ“ÓØžíÛ·K’Xÿ 0I§N$ <|ø°Éil˵k”$…„„˜˜n‚І]»©<|ø°NŸ>mrÛqòäIÅÇÇK’ºtébrÀ~]ÿý÷í·ßš˜Äö|óÍ7’Š§7oÞÜä4ðÛ(mØÃ?,I***ÒâÅ‹MNc;-ZdÎxís ü…††Z××[´h‘ÉilÇÁƒµsçNIÒ Aƒd±XLN¿‚ІuëÖM 6”$Í™3G™™™æ²—/_Ö¼yó$¯ëÕµkWsvÌÙÙY>ú¨$éÀZ¿~½É‰lÃo¼aýø‰'ž01 ”¡ spp°Þ\¦¤¤èí·ß69‘ùÞzë-ëÆO>ù$#s“=Úº¡Ò¤I“TXXhr"síÝ»WË–-“$µk×N:t09ÜšÅ`ëI›vùòe5kÖLçÏŸWÕªUµgÏ»]Ï*>>^íÛ·Wvv¶¼½½uìØ1¹¹¹™ °{£FÒ’%K$Iï½÷žž{î9“™£°°P]ºt±nP²aÃýéO29Ü#m\õêÕ5eÊIRVV–†®¼¼úÈäDPz„€ƒƒƒ>ùäÕ®][’4oÞ<½öÚk&§*?ÿ÷ÿ§ùóçK’¼¼¼´dÉÖlÌÓO?­ððpIÒ‘#GÔ¯_?ëˆß»]ll¬† b]qÞ¼yÖ ¦ "  ¬ êÖ­«¨¨(U©RERqiöæ›ošœêΛ1c†µ ­R¥ŠÖ¬Y#ooo“S¸™Å‹«cÇŽ’ŠK³ððp]¹rÅäTwÖÎ;Õ·o_k:yòd :ÔäTp{ؤ¤‚‰ŠŠÒÃ?l]‡ðùçŸ×;ï¼cÝEônQXX¨ñãÇköìÙ’$}ýõ×z衇LNà·\¸pA;wÖ±cÇ$IÁÁÁZ¿~ý]Yì¯[·NC‡UVV–$iìرš3gŽÉ©àöÝ]­’èׯŸ¢¢¢T­Z5IÅ;††‡‡+55ÕädeçÂ… ·–ƒÕªUÓºuë(  víÚŠ‰‰Qûöí%I{÷îÕ=÷Ü£mÛ¶™œ¬ìé7ÞЀ¬åà /¼ ÷ßßädðûPV@>ø 6oÞ,___IÒÚµkÕ¦MEEE™œìûú믬uëÖI’|}}£=z˜œ @iy{{ëÇT¯^½$IgÎœQ×®]5yòdåä䘜î9~ü¸þô§?iÒ¤I*((“““"##õæ›o²6*€ ‹‚°‚jß¾½öîÝ«Þ½{K’’’’Ô¿…‡‡ëäÉ“æ†ûNœ8¡~ýú)<<\III’¤>}úhß¾}jÛ¶­ÉéÜ.777­]»VÓ§O—“““ 4mÚ4iýúõfÇ»m999š2eŠZ·n­ï¿ÿ^RñîÍ›7oÖ³Ï>kr:øc(+°ÚµkkíÚµzçwäêê*©x^‹-4vìX9sÆä„·vêÔ)=õÔS ÐÚµk%I®®®š9s¦¢¢¢T«V-“ø½4iÒ$mÙ²E7–Tüf@Ÿ>}¦Í›7›œðÖ®^½ªÈÈH5mÚTS§NµŽ€8p öîÝ«“ÀÇ&%w‰#GŽè™gžÑ?ü`=æââ¢G}TO=õ”ugQ[±sçNÍ›7OK—.µn¸"I={öÔû￯æÍ››˜@YËÌÌÔk¯½¦wß}·Ä÷|hh¨ÆŽ«Zßè°ÉÉÉZ¸p¡æΫ³gÏZׯ__3gÎÔàÁƒMLe‹‚ð.óÅ_hòäÉ:räH‰ãÁÁÁzüñÇ5hÐ ùøø˜’-11Q«V­Ò¢E‹´oß¾çüýýõꫯêá‡6%€òqèÐ!½ôÒKŠŠŠÒõ?~êÔ©£‘#Gê‘GQ‡LYÏ/;;[ß}÷>ýôS}ýõ×ÊÏÏ·ž«V­šžþyMœ8Qnnnåž î$ »Paa¡>ûì3½þúëŠ/qÎÁÁA÷ÜsÂÃæŽ;ÊÉÉéŽäÈÏÏ×®]»­5kÖ(66Vÿýå —_~YÆ “ƒ3Þ{±gÏM›6MQQQ****q®~ýú8p zôè¡yxxܱÇWLLŒÖ¯_¯ 6(;;»Äywww=óÌ3š0a‚jÔ¨qÇr€™(ïb†ahóæÍúðÃõÕW_•˜ÖwMÕªUuß}÷©]»v PPP4hpÛkÿ]¸pA§NÒÁƒ§={ö(66VYYY7<ÖÕÕUÔ“O>©®]»²ó'`Ç´`Á-\¸P)))7œwppPëÖ­Õ±cG*00PÍ›7—··÷m½¹‘­3gÎ(..Nqqq:pà€¶mÛVbúðõ:vì¨1cÆhØ°aªZµêï~}PPÚ‰ÔÔT}ýõ×Z¹r¥~øáåææþæã]]]U·n]Õ¬YSžžž²X,ÖQ<—.]’aJOOWjjª’““Kõ÷õèÑCƒVÿþýU³fÍ2{m*¾üü|mÞ¼Y+W®ÔêÕ«uþüùß|¼ƒƒƒ¼¼¼T§NyxxÈÙÙYnnnrvvVvv¶rss•••¥ŒŒ %%%)##ã–‚ƒƒ5hÐ `_ - `Ubuntu `_ - `Debian `_ Or you can build your own binaries from sources. You can fetch all sources from git (below you can find all commands you need) or you can download it as tarballs at: - `libestr `_ - `liblognorm `_ Please note if you compile it from tarballs then you have to do the same steps which are mentioned below, apart from:: $ git clone ... $ autoreconf -vfi Building from git ----------------- To build liblognorm from sources, you need to have `json-c `_ installed. Open a terminal and switch to the folder where you want to build liblognorm. Below you will find the necessary commands. First, build and install prerequisite library called **libestr**:: $ git clone git@github.com:rsyslog/libestr.git $ cd libestr $ autoreconf -vfi $ ./configure $ make $ sudo make install leave that folder and repeat this step again for liblognorm:: $ cd .. $ git clone git@github.com:rsyslog/liblognorm.git $ cd liblognorm $ autoreconf -vfi $ ./configure $ make $ sudo make install That’s all you have to do. Testing ------- For a first test we need two further things, a test log and the rulebase. Both can be downloaded `here `_. After downloading these examples you can use liblognorm. Go to liblognorm/src and use the command below:: $ ./lognormalize -r messages.sampdb -o json ` tool to debug. liblognorm-2.0.8/doc/introduction.rst000066400000000000000000000030441511425433100177050ustar00rootroot00000000000000Introduction ============ Briefly described, liblognorm is a tool to normalize log data. People who need to take a look at logs often have a common problem. Logs from different machines (from different vendors) usually have different formats. Even if it is the same type of log (e.g. from firewalls), the log entries are so different, that it is pretty hard to read these. This is where liblognorm comes into the game. With this tool you can normalize all your logs. All you need is liblognorm and its dependencies and a sample database that fits the logs you want to normalize. So, for example, if you have traffic logs from three different firewalls, liblognorm will be able to "normalize" the events into generic ones. Among others, it will extract source and destination ip addresses and ports and make them available via well-defined fields. As the end result, a common log analysis application will be able to work on that common set and so this backend will be independent from the actual firewalls feeding it. Even better, once we have a well-understood interim format, it is also easy to convert that into any other vendor specific format, so that you can use that vendor's analysis tool. By design, liblognorm is constructed as a library. Thus, it can be used by other tools. In short, liblognorm works by: 1. Matching a line to a rule from predefined configuration; 2. Picking out variable fields from the line; 3. Returning them as a JSON hash object. Then, a consumer of this object can construct new, normalized log line on its own. liblognorm-2.0.8/doc/libraryapi.rst000066400000000000000000000004431511425433100173220ustar00rootroot00000000000000Library API =========== To use the library, include liblognorm.h (which is quoted below) into your code. The API is fairly simple and hardly needs further explanations. .. literalinclude:: ../src/liblognorm.h :start-after: #define LIBLOGNORM_H_INCLUDED :end-before: #endif :language: c liblognorm-2.0.8/doc/license.rst000066400000000000000000000002201511425433100165770ustar00rootroot00000000000000Licensing ========= Liblognorm is available under the terms of the GNU LGPL v2.1 or above (full text below). .. literalinclude:: ../COPYING liblognorm-2.0.8/doc/lognormalizer.rst000066400000000000000000000171371511425433100200600ustar00rootroot00000000000000Lognormalizer ============= Lognormalizer is a sample tool which is often used to test and debug rulebases before real use. Nevertheless, it can be used in production as a simple command line interface to liblognorm. This tool reads log lines from its standard input and prints results to standard output. You need to use redirections if you want to read or write files. An example of the command:: $ lognormalizer -r messages.sampdb -e json Specifies name of the file containing the rulebase. :: -v Increase verbosity level. Can be used several times. If used three times, internal data structures are dumped (make sense to developers, only). :: -p Print only successfully parsed messages. :: -P Print only messages **not** successfully parsed. :: -L Add line number information to events not successfully parsed. This is meant as a troubleshooting aid when working with unparsable events, as the information can be used to directly go to the line in question in the source data file. The line number is contained in a field named ``lognormalizer.line_nbr``. :: -t Print only those messages which have this tag. :: -T Include 'event.tags' attribute when output is in JSON format. This attribute contains list of tags of the matched rule. :: -E Encoder-specific data. For CSV, it is the list of fields to be output, separated by comma or space. It is currently unused for other formats. :: -d Generate DOT file describing parse tree. It is used to plot parse graph with GraphViz. :: -H At end of run, print a summary line with number of messages processed, parsed and unparsed to stdout. :: -U At end of run, print a summary line with number of messages unparsed to stdout. Note that this message is only printed if there was at least one unparsable message. :: -o Special options. The following ones can be set: * **allowRegex** Permits to use regular expressions inse the v1 engine This is deprecated and should not be used for new deployments. * **addExecPath** Includes metadata into the event on how it was (tried) to be parsed. Can be useful in troubleshooting normalization problems. * **addOriginalMsg** Always add the "original-msg" data item. By default, this is only done when a message could not be parsed. * **addRule** Add a mockup of the rule that was processed. Note that it is *not* an exact copy of the rule, but a rule that correctly describes the parsed message. Most importantly, prefixes are appended and custom data types are expanded (and no longer visible as such). This option is primarily meant for postprocessing, e.g. as input to an anonymizer. * **addRuleRulcation** For rules that successfully parsed, add the location of the rule inside the rulebase. But the file name as well as the line number are given. If two rules evaluate to the same end node, only a single rule location is given. However, in practice this is extremely unlikely and as such for practical reasons the information can be considered reliable. :: -s At end of run, print internal parse DAG statistics and exit. This option is meant for developers and researches which want to get insight into the quality of the algorithm and/or how efficient the rulebase could be processed. **NOT** intended for end users. This option is performance intense. :: -S Even stronger statistics than -s. Requires that the version is compiled with --enable-advanced-statistics, which causes a considerable performance loss. :: -x Print statistics as a DOT file. In order to keep the graph readable, information is only emitted for called nodes. :: -e Output format. By default, output is in JSON format. With this option, you can change it to a different one. Supported Output Formats ........................ The JSON, XML, and CSV formats should be self-explanatory. The cee-syslog format emits messages according to the Mitre CEE spec. Note that the cee-syslog format is primarily supported for backward-compatibility. It does **not** support nested data items and as such cannot be used when the rulebase makes use of this feature (we assume this most often happens nowadays). We strongly recommend not use it for new deployments. Support may be removed in later releases. The raw format outputs an exact copy of the input message, without any normalization visible. The prime use case of "raw" is to extract either all messages that could or could not be normalized. To do so specify the -p or -P option. Also, it works in combination with the -t option to extract a subset based on tagging. In any case, the core use is to prepare a subset of the original file for further processing. Examples -------- These examples were created using sample rulebase from source package. Default (CEE) output:: $ lognormalizer -r rulebases/sample.rulebase Weight: 42kg [cee@115 event.tags="tag2" unit="kg" N="42" fat="free"] Snow White and the Seven Dwarfs [cee@115 event.tags="tale" company="the Seven Dwarfs"] 2012-10-11 src=127.0.0.1 dst=88.111.222.19 [cee@115 dst="88.111.222.19" src="127.0.0.1" date="2012-10-11"] JSON output, flat tags enabled:: $ lognormalizer -r rulebases/sample.rulebase -e json -T %% { "event.tags": [ "tag3", "percent" ], "percent": "100", "part": "wha", "whole": "whale" } Weight: 42kg { "unit": "kg", "N": "42", "event.tags": [ "tag2" ], "fat": "free" } CSV output with fixed field list:: $ lognormalizer -r rulebases/sample.rulebase -e csv -E'N unit' Weight: 42kg "42","kg" Weight: 115lbs "115","lbs" Anything not matching the rule , Creating a graph of the rulebase -------------------------------- To get a better overview of a rulebase you can create a graph that shows you the chain of normalization (parse-tree). At first you have to install an additional package called graphviz. Graphviz is a tool that creates such a graph with the help of a control file (created with the rulebase). `Here `_ you will find more information about graphviz. To install it you can use the package manager. For example, on RedHat systems it is yum command:: $ sudo yum install graphviz The next step would be creating the control file for graphviz. Therefore we use the normalizer command with the options -d "preferred filename for the control file" and -r "rulebase":: $ lognormalize -d control.dot -r messages.rb Please note that there is no need for an input or output file. If you have a look at the control file now you will see that the content is a little bit confusing, but it includes all information, like the nodes, fields and parser, that graphviz needs to create the graph. Of course you can edit that file, but please note that it is a lot of work. Now we can create the graph by typing:: $ dot control.dot -Tpng >graph.png dot + name of control file + option -T -> file format + output file That is just one example for using graphviz, of course you can do many other great things with it. But I think this "simple" graph could be very helpful for the normalizer. Below you see sample for such a graph, but please note that this is not such a pretty one. Such a graph can grow very fast by editing your rulebase. .. figure:: graph.png :width: 90 % :alt: graph sample liblognorm-2.0.8/doc/sample_rulebase.rst000066400000000000000000000001411511425433100203220ustar00rootroot00000000000000Sample rulebase =============== .. literalinclude:: ../rulebases/sample.rulebase :linenos: liblognorm-2.0.8/lognorm.pc.in000066400000000000000000000004401511425433100162700ustar00rootroot00000000000000prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ Name: lognorm Description: fast samples-based log normalization library Version: @VERSION@ Requires: libfastjson Libs: -L${libdir} -llognorm Libs.private: @pkg_config_libs_private@ Cflags: -I${includedir} liblognorm-2.0.8/m4/000077500000000000000000000000001511425433100142045ustar00rootroot00000000000000liblognorm-2.0.8/m4/dummy000066400000000000000000000000001511425433100152500ustar00rootroot00000000000000liblognorm-2.0.8/rulebases/000077500000000000000000000000001511425433100156515ustar00rootroot00000000000000liblognorm-2.0.8/rulebases/cisco.rulebase000066400000000000000000000012461511425433100205000ustar00rootroot00000000000000prefix=%date:date-rfc3164% %host:word% %seqnum:number%: %othseq:char-to:\x3a%: %%%tag:char-to:\x3a%: rule=: Configured from console by %tty:word:% (%ip:ipv4%) rule=: Authentication failure for %proto:word% req from host %ip:ipv4% rule=: Interface %interface:char-to:,%, changed state to %state:word% rule=: Line protocol on Interface %interface:char-to:,%, changed state to %state:word% rule=: Attempted to connect to %servname:word% from %ip:ipv4% # too-generic syntaces (like %port:word% below) cause problems. # Best is to have very specific syntaxes, but as an # interim solution we may need to backtrack if there is no other way to handle it. #: %port:word% transmit error liblognorm-2.0.8/rulebases/messages.rulebase000066400000000000000000000011441511425433100212040ustar00rootroot00000000000000prefix=%date:date-rfc3164% %host:word% %tag:char-to:\x3a%: rule=: restart. rule=: Bad line received from identity server at %ip:ipv4%: %port:number% rule=: FTP session closed rule=: wu-ftpd - TLS settings: control %wuftp-control:char-to:,%, client_cert %wuftp-clcert:char-to:,%, data %wuftp-allow:word% rule=: User %user:word% timed out after %timeout:number% seconds at %otherdatesyntax:word% %otherdate:date-rfc3164% %otheryear:word% rule=: getpeername (in.ftpd): Transport endpoint is not connected # the one below is problematic (and needs some backtracking) #: %disk:char-to:\x3a%: timeout waiting for DMA liblognorm-2.0.8/rulebases/sample.rulebase000066400000000000000000000036411511425433100206620ustar00rootroot00000000000000# Some sample rules and strings matching them # Prefix sample: # myhostname: code=23 prefix=%host:char-to:\x3a%: rule=prefixed_code:code=%code:number% # myhostname: name=somename rule=prefixed_name:name=%name:word% # Reset prefix to default (empty value): prefix= # Quantity: 555 rule=tag1:Quantity: %N:number% # Weight: 42kg rule=tag2:Weight: %N:number%%unit:word% annotate=tag2:+fat="free" # %% rule=tag3,percent:\x25%% annotate=percent:+percent="100" annotate=tag3:+whole="whale" annotate=tag3:+part="wha" # literal rule=tag4,tag5,tag6,tag4:literal annotate=tag4:+this="that" # first field,second field,third field,fourth field rule=csv:%r1:char-to:,%,%r2:char-to:,%,%r3:char-to:,%,%r4:rest% # CSV: field1,,field3 rule=better-csv:CSV: %f1:char-sep:,%,%f2:char-sep:,%,%f3:char-sep:,% # Snow White and the Seven Dwarfs rule=tale:Snow White and %company:rest% # iptables: SRC=192.168.1.134 DST=46.252.161.13 LEN=48 TOS=0x00 PREC=0x00 rule=ipt:iptables: %dummy:iptables% # 2012-10-11 src=127.0.0.1 dst=88.111.222.19 rule=:%date:date-iso% src=%src:ipv4% dst=%dst:ipv4% # Oct 29 09:47:08 server rsyslogd: rsyslogd's groupid changed to 103 rule=syslog:%date1:date-rfc3164% %host:word% %tag:char-to:\x3a%: %text:rest% # Oct 29 09:47:08 rule=rfc3164:%date1:date-rfc3164% # 1985-04-12T19:20:50.52-04:00 rule=rfc5424:%date1:date-rfc5424% # 1985-04-12T19:20:50.52-04:00 testing 123 rule=rfc5424:%date1:date-rfc5424% %test:word% %test2:number% # quoted_string="Contents of a quoted string cannot include quote marks" rule=quote:quoted_string=%quote:quoted-string% # tokenized words: aaa.org; bbb.com; ccc.net rule=tokenized_words:tokenized words: %arr:tokenized:; :char-sep:\x3b% # tokenized regex: aaa.org; bbb.com; ccc.net rule=tokenized_regex:tokenized regex: %arr:tokenized:; :regex:[^; ]+% # regex: abcdef rule=regex:regex: %token:regex:abc.ef% # host451 # generates { basename:"host", hostid:451 } rule=:%basename:alpha%%hostid:number% liblognorm-2.0.8/rulebases/syntax.txt000066400000000000000000000103621511425433100177420ustar00rootroot00000000000000WARNING ======= This file is somewhat obsolete, for current information look at doc/ directory. Basic syntax ============ Each line in rulebase file is evaluated separately. Lines starting with '#' are commentaries. Empty lines are just skipped, they can be inserted for readability. If the line starts with 'rule=', then it contains a rule. This line has following format: rule=[[,...]]: Everything before a colon is treated as comma-separated list of tags, which will be attached to a match. After the colon, match description should be given. It consists of string literals and field selectors. String literals should match exactly. Field selector has this format: %:[:]% Percent sign is used to enclose field selector. If you need to match literal '%', it can be written as '%%' or '\x25'. Behaviour of field selector depends on its type, which is described below. If field name is set to '-', this field is matched but not saved. Several rules can have a common prefix. You can set it once with this syntax: prefix= Every following rule will be treated as an addition to this prefix. Prefix can be reset to default (empty value) by the line: prefix= Tags of the matched rule are attached to the message and can be used to annotate it. Annotation allows to add fixed fields to the message. Syntax is as following: annotate=:+="" Field value should always be enclosed in double quote marks. There can be multiple annotations for the same tag. Field types =========== Field type: 'number' Matches: One or more decimal digits. Extra data: Not used Example: %field_name:number% Field type: 'word' Matches: One or more characters, up to the next space (\x20), or up to end of line. Extra data: Not used Example: %field_name:word% Field type: 'alpha' Matches: One or more alphabetic characters, up to the next whitespace, punctuation, decimal digit or ctrl. Extra data: Not used Example: %field_name:alpha% Field type: 'char-to' Matches: One or more characters, up to the next character given in extra data. Extra data: One character (can be escaped) Example: %field_name:char-to:,% %field_name:char-to:\x25% Field type: 'char-sep' Matches: Zero or more characters, up to the next character given in extra data, or up to end of line. Extra data: One character (can be escaped) Example: %field_name:char-sep:,% %field_name:char-sep:\x25% Field type: 'rest' Matches: Zero or more characters till end of line. Extra data: Not used Example: %field_name:rest% Notes: Should be always at end of the rule. Field type: 'quoted-string' Matches: Zero or more characters, surrounded by double quote marks. Extra data: Not used Example: %field_name:quoted-string% Notes: Quote marks are stripped from the match. Field type: 'date-iso' Matches: Date of format 'YYYY-MM-DD'. Extra data: Not used Example: %field-name:date-iso% Field type: 'time-24hr' Matches: Time of format 'HH:MM:SS', where HH is 00..23. Extra data: Not used Example: %field_name:time-24hr% Field type: 'time-12hr' Matches: Time of format 'HH:MM:SS', where HH is 00..12. Extra data: Not used Example: %field_name:time-12hr% Field type: 'ipv4' Matches: IPv4 address, in dot-decimal notation (AAA.BBB.CCC.DDD). Extra data: Not used Example: %field_name:ipv4% Field type: 'date-rfc3164' Matches: Valid date/time in RFC3164 format, i.e.: 'Oct 29 09:47:08' Extra data: Not used Example: %field_name:date-rfc3164% Notes: This parser implements several quirks to match malformed timestamps from some devices. Field type: 'date-rfc5424' Matches: Valid date/time in RFC5424 format, i.e.: '1985-04-12T19:20:50.52-04:00' Extra data: Not used Example: %field_name:date-rfc5424% Notes: Slightly different formats are allowed. Field type: 'iptables' Matches: Name=value pairs, separated by spaces, as in Netfilter log messages. Extra data: Not used Example: %-:iptables% Notes: Name of the selector is not used; names from the line are used instead. This selector always matches everything till end of the line. Cannot match zero characters. Examples ======== Look at sample.rulebase for example rules and matching lines. liblognorm-2.0.8/src/000077500000000000000000000000001511425433100144535ustar00rootroot00000000000000liblognorm-2.0.8/src/.gitignore000066400000000000000000000001171511425433100164420ustar00rootroot00000000000000*.o *.lo *.la Makefile Makefile.in .deps .libs lognormalizer lognorm-features.hliblognorm-2.0.8/src/Makefile.am000066400000000000000000000036421511425433100165140ustar00rootroot00000000000000# Uncomment for debugging DEBUG = -g PTHREADS_CFLAGS = -pthread #CFLAGS += $(DEBUG) # we need to clean the normalizer up once we have reached a decent # milestone (latest at initial release!) bin_PROGRAMS = lognormalizer lognormalizer_SOURCES = lognormalizer.c lognormalizer_CPPFLAGS = -I$(top_srcdir) $(WARN_CFLAGS) $(JSON_C_CFLAGS) $(LIBESTR_CFLAGS) lognormalizer_LDADD = $(JSON_C_LIBS) $(LIBLOGNORM_LIBS) $(LIBESTR_LIBS) ../compat/compat.la lognormalizer_DEPENDENCIES = liblognorm.la check_PROGRAMS = ln_test ln_test_SOURCES = $(lognormalizer_SOURCES) ln_test_CPPFLAGS = $(lognormalizer_CPPFLAGS) ln_test_LDADD = $(lognormalizer_LDADD) ln_test_DEPENDENCIES = $(lognormalizer_DEPENDENCIES) ln_test_LDFLAGS = -no-install lib_LTLIBRARIES = liblognorm.la liblognorm_la_SOURCES = \ liblognorm.c \ pdag.c \ annot.c \ samp.c \ lognorm.c \ parser.c \ enc_syslog.c \ enc_csv.c \ enc_xml.c # Users violently requested that v2 shall be able to understand v1 # rulebases. As both are very very different, we now include the # full v1 engine for this purpose. This here is what does this. # see also: https://github.com/rsyslog/liblognorm/issues/103 liblognorm_la_SOURCES += \ v1_liblognorm.c \ v1_parser.c \ v1_ptree.c \ v1_samp.c liblognorm_la_CPPFLAGS = $(JSON_C_CFLAGS) $(WARN_CFLAGS) $(LIBESTR_CFLAGS) $(PCRE_CFLAGS) liblognorm_la_LIBADD = $(rt_libs) $(JSON_C_LIBS) $(LIBESTR_LIBS) $(PCRE_LIBS) -lestr # info on version-info: # http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html # Note: v2 now starts at version 5, as v1 previously also had 4 liblognorm_la_LDFLAGS = -version-info 6:0:1 EXTRA_DIST = \ internal.h \ liblognorm.h \ lognorm.h \ pdag.h \ annot.h \ samp.h \ enc.h \ parser.h \ helpers.h # and now the old cruft: EXTRA_DIST += \ v1_liblognorm.h \ v1_parser.h \ v1_samp.h \ v1_ptree.h include_HEADERS = liblognorm.h samp.h lognorm.h pdag.h annot.h enc.h parser.h lognorm-features.h liblognorm-2.0.8/src/annot.c000066400000000000000000000123101511425433100157330ustar00rootroot00000000000000/** * @file annot.c * @brief Implementation of the annotation set object. * @class ln_annot annot.h *//* * Copyright 2011 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Sannott, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include #include #include #include #include #include "lognorm.h" #include "samp.h" #include "annot.h" #include "internal.h" ln_annotSet* ln_newAnnotSet(ln_ctx ctx) { ln_annotSet *as; if((as = calloc(1, sizeof(struct ln_annotSet_s))) == NULL) goto done; as->ctx = ctx; done: return as; } void ln_deleteAnnotSet(ln_annotSet *as) { ln_annot *node, *nextnode; if(as == NULL) goto done; for(node = as->aroot; node != NULL; node = nextnode) { nextnode = node->next; ln_deleteAnnot(node); } free(as); done: return; } ln_annot* ln_findAnnot(ln_annotSet *as, es_str_t *tag) { ln_annot *annot; if(as == NULL) { annot = NULL; goto done; } for( annot = as->aroot ; annot != NULL && es_strcmp(annot->tag, tag) ; annot = annot->next) { ; /* do nothing, just search... */ } done: return annot; } /** * Combine two annotations. * @param[in] annot currently existing and surviving annotation * @param[in] add annotation to be added. This will be destructed * as part of the process. * @returns 0 if ok, something else otherwise */ static int ln_combineAnnot(ln_annot *annot, ln_annot *add) { int r = 0; ln_annot_op *op, *nextop; for(op = add->oproot; op != NULL; op = nextop) { CHKR(ln_addAnnotOp(annot, op->opc, op->name, op->value)); nextop = op->next; free(op); } es_deleteStr(add->tag); free(add); done: return r; } int ln_addAnnotToSet(ln_annotSet *as, ln_annot *annot) { int r = 0; ln_annot *aexist; assert(annot->tag != NULL); aexist = ln_findAnnot(as, annot->tag); if(aexist == NULL) { /* does not yet exist, simply store new annot */ annot->next = as->aroot; as->aroot = annot; } else { /* annotation already exists, combine */ r = ln_combineAnnot(aexist, annot); } return r; } ln_annot* ln_newAnnot(es_str_t *tag) { ln_annot *annot; if((annot = calloc(1, sizeof(struct ln_annot_s))) == NULL) goto done; annot->tag = tag; done: return annot; } void ln_deleteAnnot(ln_annot *annot) { ln_annot_op *op, *nextop; if(annot == NULL) goto done; es_deleteStr(annot->tag); for(op = annot->oproot; op != NULL; op = nextop) { es_deleteStr(op->name); if(op->value != NULL) es_deleteStr(op->value); nextop = op->next; free(op); } free(annot); done: return; } int ln_addAnnotOp(ln_annot *annot, ln_annot_opcode opc, es_str_t *name, es_str_t *value) { int r = -1; ln_annot_op *node; if((node = calloc(1, sizeof(struct ln_annot_op_s))) == NULL) goto done; node->opc = opc; node->name = name; node->value = value; if(annot->oproot != NULL) { node->next = annot->oproot; } annot->oproot = node; r = 0; done: return r; } /* annotate the event with a specific tag. helper to keep code * small and easy to follow. */ static inline int ln_annotateEventWithTag(ln_ctx ctx, struct json_object *json, es_str_t *tag) { int r=0; ln_annot *annot; ln_annot_op *op; struct json_object *field; char *cstr; if (NULL == (annot = ln_findAnnot(ctx->pas, tag))) goto done; for(op = annot->oproot ; op != NULL ; op = op->next) { if(op->opc == ln_annot_ADD) { CHKN(cstr = ln_es_str2cstr(&op->value)); CHKN(field = json_object_new_string(cstr)); CHKN(cstr = ln_es_str2cstr(&op->name)); json_object_object_add(json, cstr, field); } else { // TODO: implement } } done: return r; } int ln_annotate(ln_ctx ctx, struct json_object *json, struct json_object *tagbucket) { int r = 0; es_str_t *tag; struct json_object *tagObj; const char *tagCstr; int i; ln_dbgprintf(ctx, "ln_annotate called [aroot=%p]", ctx->pas->aroot); /* shortcut: terminate immediately if nothing to do... */ if(ctx->pas->aroot == NULL) goto done; /* iterate over tagbucket */ for (i = json_object_array_length(tagbucket) - 1; i >= 0; i--) { CHKN(tagObj = json_object_array_get_idx(tagbucket, i)); CHKN(tagCstr = json_object_get_string(tagObj)); ln_dbgprintf(ctx, "ln_annotate, current tag %d, cstr %s", i, tagCstr); CHKN(tag = es_newStrFromCStr(tagCstr, strlen(tagCstr))); CHKR(ln_annotateEventWithTag(ctx, json, tag)); es_deleteStr(tag); } done: return r; } liblognorm-2.0.8/src/annot.h000066400000000000000000000115261511425433100157500ustar00rootroot00000000000000/** * @file annot.h * @brief The annotation set object * @class ln_annot annot.h *//* * Copyright 2011 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is meant to be included by applications using liblognorm. * For lognorm library files themselves, include "lognorm.h". * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef LIBLOGNORM_ANNOT_H_INCLUDED #define LIBLOGNORM_ANNOT_H_INCLUDED #include typedef struct ln_annotSet_s ln_annotSet; typedef struct ln_annot_s ln_annot; typedef struct ln_annot_op_s ln_annot_op; typedef enum {ln_annot_ADD=0, ln_annot_RM=1} ln_annot_opcode; /** * List of annotation operations. */ struct ln_annot_op_s { ln_annot_op *next; ln_annot_opcode opc; /**< opcode */ es_str_t *name; es_str_t *value; }; /** * annotation object */ struct ln_annot_s { ln_annot *next; /**< used for chaining annotations */ es_str_t *tag; /**< tag associated for this annotation */ ln_annot_op *oproot; }; /** * annotation set object * * Note: we do not (yet) use a hash table. However, performance should * be gained by pre-processing rules so that tags directly point into * the annotation. This is even faster than hash table access. */ struct ln_annotSet_s { ln_annot *aroot; ln_ctx ctx; /**< save our context for easy dbgprintf et al... */ }; /* Methods */ /** * Allocates and initializes a new annotation set. * @memberof ln_annot * * @param[in] ctx current library context. This MUST match the * context of the parent. * * @return pointer to new node or NULL on error */ ln_annotSet* ln_newAnnotSet(ln_ctx ctx); /** * Free annotation set and destruct all members. * @memberof ln_annot * * @param[in] tree pointer to annot to free */ void ln_deleteAnnotSet(ln_annotSet *as); /** * Find annotation inside set based on given tag name. * @memberof ln_annot * * @param[in] as annotation set * @param[in] tag tag name to look for * * @returns NULL if not found, ptr to object otherwise */ ln_annot* ln_findAnnot(ln_annotSet *as, es_str_t *tag); /** * Add annotation to set. * If an annotation associated with this tag already exists, these * are combined. If not, a new annotation is added. Note that the * caller must not access any of the objects passed in to this method * after it has finished (objects may become deallocated during the * method). * @memberof ln_annot * * @param[in] as annotation set * @param[in] annot annotation to add * * @returns 0 on success, something else otherwise */ int ln_addAnnotToSet(ln_annotSet *as, ln_annot *annot); /** * Allocates and initializes a new annotation. * The tag passed in identifies the new annotation. The caller * no longer owns the tag string after calling this method, so * it must not access the same copy when the method returns. * @memberof ln_annot * * @param[in] tag tag associated to annot (must not be NULL) * @return pointer to new node or NULL on error */ ln_annot* ln_newAnnot(es_str_t *tag); /** * Free annotation and destruct all members. * @memberof ln_annot * * @param[in] tree pointer to annot to free */ void ln_deleteAnnot(ln_annot *annot); /** * Add an operation to the annotation set. * The operation description will be added as entry. * @memberof ln_annot * * @param[in] annot pointer to annot to modify * @param[in] op operation * @param[in] name name of field, must NOT be re-used by caller * @param[in] value value of field, may be NULL (e.g. in remove operation), * must NOT be re-used by caller * @returns 0 on success, something else otherwise */ int ln_addAnnotOp(ln_annot *annot, ln_annot_opcode opc, es_str_t *name, es_str_t *value); /** * Annotate an event. * This adds annotations based on the event's tagbucket. * @memberof ln_annot * * @param[in] ctx current context * @param[in] event event to annotate (updated with annotations on exit) * @returns 0 on success, something else otherwise */ int ln_annotate(ln_ctx ctx, struct json_object *json, struct json_object *tags); #endif /* #ifndef LOGNORM_ANNOT_H_INCLUDED */ liblognorm-2.0.8/src/enc.h000066400000000000000000000026361511425433100154000ustar00rootroot00000000000000/** * @file enc.h * @brief Encoder functions */ /* * liblognorm - a fast samples-based log normalization library * Copyright 2010 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef LIBLOGNORM_ENC_H_INCLUDED #define LIBLOGNORM_ENC_H_INCLUDED int ln_fmtEventToRFC5424(struct json_object *json, es_str_t **str); int ln_fmtEventToCSV(struct json_object *json, es_str_t **str, es_str_t *extraData); int ln_fmtEventToXML(struct json_object *json, es_str_t **str); #endif /* LIBLOGNORM_ENC_H_INCLUDED */ liblognorm-2.0.8/src/enc_csv.c000066400000000000000000000127011511425433100162400ustar00rootroot00000000000000/** * @file enc_csv.c * Encoder for CSV format. Note: CEE currently think about what a * CEE-compliant CSV format may look like. As such, the format of * this output will most probably change once the final decision * has been made. At this time (2010-12), I do NOT even try to * stay inline with the discussion. * * This file contains code from all related objects that is required in * order to encode this format. The core idea of putting all of this into * a single file is that this makes it very straightforward to write * encoders for different encodings, as all is in one place. * */ /* * liblognorm - a fast samples-based log normalization library * Copyright 2010-2018 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include #include #include #include #include "lognorm.h" #include "internal.h" #include "enc.h" static char hexdigit[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; /* TODO: CSV encoding for Unicode characters is as of RFC4627 not fully * supported. The algorithm is that we must build the wide character from * UTF-8 (if char > 127) and build the full 4-octet Unicode character out * of it. Then, this needs to be encoded. Currently, we work on a * byte-by-byte basis, which simply is incorrect. * rgerhards, 2010-11-09 */ static int ln_addValue_CSV(const char *buf, es_str_t **str) { int r; unsigned char c; es_size_t i; char numbuf[4]; int j; assert(str != NULL); assert(*str != NULL); assert(buf != NULL); for(i = 0; i < strlen(buf); i++) { c = buf[i]; if((c >= 0x23 && c <= 0x5b) || (c >= 0x5d /* && c <= 0x10FFFF*/) || c == 0x20 || c == 0x21) { /* no need to escape */ es_addChar(str, c); } else { /* we must escape, try RFC4627-defined special sequences first */ switch(c) { case '\0': es_addBuf(str, "\\u0000", 6); break; case '\"': es_addBuf(str, "\\\"", 2); break; case '\\': es_addBuf(str, "\\\\", 2); break; case '\010': es_addBuf(str, "\\b", 2); break; case '\014': es_addBuf(str, "\\f", 2); break; case '\n': es_addBuf(str, "\\n", 2); break; case '\r': es_addBuf(str, "\\r", 2); break; case '\t': es_addBuf(str, "\\t", 2); break; default: /* TODO : proper Unicode encoding (see header comment) */ for(j = 0 ; j < 4 ; ++j) { numbuf[3-j] = hexdigit[c % 16]; c = c / 16; } es_addBuf(str, "\\u", 2); es_addBuf(str, numbuf, 4); break; } } } r = 0; return r; } static int ln_addField_CSV(struct json_object *field, es_str_t **str) { int r, i; struct json_object *obj; int needComma = 0; const char *value; assert(field != NULL); assert(str != NULL); assert(*str != NULL); switch(json_object_get_type(field)) { case json_type_array: CHKR(es_addChar(str, '[')); for (i = json_object_array_length(field) - 1; i >= 0; i--) { if(needComma) es_addChar(str, ','); else needComma = 1; CHKN(obj = json_object_array_get_idx(field, i)); CHKN(value = json_object_get_string(obj)); CHKR(ln_addValue_CSV(value, str)); } CHKR(es_addChar(str, ']')); break; case json_type_string: case json_type_int: CHKN(value = json_object_get_string(field)); CHKR(ln_addValue_CSV(value, str)); break; case json_type_null: case json_type_boolean: case json_type_double: case json_type_object: CHKR(es_addBuf(str, "***unsupported type***", sizeof("***unsupported type***")-1)); break; default: CHKR(es_addBuf(str, "***OBJECT***", sizeof("***OBJECT***")-1)); } r = 0; done: return r; } int ln_fmtEventToCSV(struct json_object *json, es_str_t **str, es_str_t *extraData) { int r = -1; int needComma = 0; struct json_object *field; char *namelist = NULL, *name, *nn; assert(json != NULL); assert(json_object_is_type(json, json_type_object)); if((*str = es_newStr(256)) == NULL) goto done; if(extraData == NULL) goto done; CHKN(namelist = es_str2cstr(extraData, NULL)); for (name = namelist; name != NULL; name = nn) { for (nn = name; *nn != '\0' && *nn != ',' && *nn != ' '; nn++) { /* do nothing */ } if (*nn == '\0') { nn = NULL; } else { *nn = '\0'; nn++; } json_object_object_get_ex(json, name, &field); if (needComma) { CHKR(es_addChar(str, ',')); } else { needComma = 1; } if (field != NULL) { CHKR(es_addChar(str, '"')); ln_addField_CSV(field, str); CHKR(es_addChar(str, '"')); } } r = 0; done: if (namelist != NULL) free(namelist); return r; } liblognorm-2.0.8/src/enc_syslog.c000066400000000000000000000123711511425433100167700ustar00rootroot00000000000000/** * @file enc_syslog.c * Encoder for syslog format. * This file contains code from all related objects that is required in * order to encode syslog format. The core idea of putting all of this into * a single file is that this makes it very straightforward to write * encoders for different encodings, as all is in one place. */ /* * liblognorm - a fast samples-based log normalization library * Copyright 2010-2016 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include #include #include #include #include "internal.h" #include "liblognorm.h" #include "enc.h" static int ln_addValue_Syslog(const char *value, es_str_t **str) { int r; es_size_t i; assert(str != NULL); assert(*str != NULL); assert(value != NULL); for(i = 0; i < strlen(value); i++) { switch(value[i]) { case '\0': es_addChar(str, '\\'); es_addChar(str, '0'); break; case '\n': es_addChar(str, '\\'); es_addChar(str, 'n'); break; /* TODO : add rest of control characters here... */ case ',': /* comma is CEE-reserved for lists */ es_addChar(str, '\\'); es_addChar(str, ','); break; #if 0 /* alternative encoding for discussion */ case '^': /* CEE-reserved for lists */ es_addChar(str, '\\'); es_addChar(str, '^'); break; #endif /* at this layer ... do we need to think about transport * encoding at all? Or simply leave it to the transport agent? */ case '\\': /* RFC5424 reserved */ es_addChar(str, '\\'); es_addChar(str, '\\'); break; case ']': /* RFC5424 reserved */ es_addChar(str, '\\'); es_addChar(str, ']'); break; case '\"': /* RFC5424 reserved */ es_addChar(str, '\\'); es_addChar(str, '\"'); break; default: es_addChar(str, value[i]); break; } } r = 0; return r; } static int ln_addField_Syslog(char *name, struct json_object *field, es_str_t **str) { int r; const char *value; int needComma = 0; struct json_object *obj; int i; assert(field != NULL); assert(str != NULL); assert(*str != NULL); CHKR(es_addBuf(str, name, strlen(name))); CHKR(es_addBuf(str, "=\"", 2)); switch(json_object_get_type(field)) { case json_type_array: for (i = json_object_array_length(field) - 1; i >= 0; i--) { if(needComma) es_addChar(str, ','); else needComma = 1; CHKN(obj = json_object_array_get_idx(field, i)); CHKN(value = json_object_get_string(obj)); CHKR(ln_addValue_Syslog(value, str)); } break; case json_type_string: case json_type_int: CHKN(value = json_object_get_string(field)); CHKR(ln_addValue_Syslog(value, str)); break; case json_type_null: case json_type_boolean: case json_type_double: case json_type_object: CHKR(es_addBuf(str, "***unsupported type***", sizeof("***unsupported type***")-1)); break; default: CHKR(es_addBuf(str, "***OBJECT***", sizeof("***OBJECT***")-1)); } CHKR(es_addChar(str, '\"')); r = 0; done: return r; } static inline int ln_addTags_Syslog(struct json_object *taglist, es_str_t **str) { int r = 0; struct json_object *tagObj; int needComma = 0; const char *tagCstr; int i; assert(json_object_is_type(taglist, json_type_array)); CHKR(es_addBuf(str, " event.tags=\"", 13)); for (i = json_object_array_length(taglist) - 1; i >= 0; i--) { if(needComma) es_addChar(str, ','); else needComma = 1; CHKN(tagObj = json_object_array_get_idx(taglist, i)); CHKN(tagCstr = json_object_get_string(tagObj)); CHKR(es_addBuf(str, (char*)tagCstr, strlen(tagCstr))); } es_addChar(str, '"'); done: return r; } int ln_fmtEventToRFC5424(struct json_object *json, es_str_t **str) { int r = -1; struct json_object *tags; assert(json != NULL); assert(json_object_is_type(json, json_type_object)); if((*str = es_newStr(256)) == NULL) goto done; es_addBuf(str, "[cee@115", 8); if(json_object_object_get_ex(json, "event.tags", &tags)) { CHKR(ln_addTags_Syslog(tags, str)); } struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { char *const name = (char*)json_object_iter_peek_name(&it); if (strcmp(name, "event.tags")) { es_addChar(str, ' '); ln_addField_Syslog(name, json_object_iter_peek_value(&it), str); } json_object_iter_next(&it); } es_addChar(str, ']'); done: return r; } liblognorm-2.0.8/src/enc_xml.c000066400000000000000000000131041511425433100162430ustar00rootroot00000000000000/** * @file enc-xml.c * Encoder for XML format. * * This file contains code from all related objects that is required in * order to encode this format. The core idea of putting all of this into * a single file is that this makes it very straightforward to write * encoders for different encodings, as all is in one place. * */ /* * liblognorm - a fast samples-based log normalization library * Copyright 2010-2016 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include #include #include #include "lognorm.h" #include "internal.h" #include "enc.h" #if 0 static char hexdigit[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; #endif /* TODO: XML encoding for Unicode characters is as of RFC4627 not fully * supported. The algorithm is that we must build the wide character from * UTF-8 (if char > 127) and build the full 4-octet Unicode character out * of it. Then, this needs to be encoded. Currently, we work on a * byte-by-byte basis, which simply is incorrect. * rgerhards, 2010-11-09 */ static int ln_addValue_XML(const char *value, es_str_t **str) { int r; unsigned char c; es_size_t i; #if 0 char numbuf[4]; int j; #endif assert(str != NULL); assert(*str != NULL); assert(value != NULL); // TODO: support other types! es_addBuf(str, "", 7); for(i = 0 ; i < strlen(value) ; ++i) { c = value[i]; switch(c) { case '\0': es_addBuf(str, "�", 5); break; #if 0 case '\n': es_addBuf(str, " ", 5); break; case '\r': es_addBuf(str, " ", 5); break; case '\t': es_addBuf(str, "&x08;", 5); break; case '\"': es_addBuf(str, """, 6); break; #endif case '<': es_addBuf(str, "<", 4); break; case '&': es_addBuf(str, "&", 5); break; #if 0 case ',': es_addBuf(str, "\\,", 2); break; case '\'': es_addBuf(str, "'", 6); break; #endif default: es_addChar(str, c); #if 0 /* TODO : proper Unicode encoding (see header comment) */ for(j = 0 ; j < 4 ; ++j) { numbuf[3-j] = hexdigit[c % 16]; c = c / 16; } es_addBuf(str, "\\u", 2); es_addBuf(str, numbuf, 4); break; #endif } } es_addBuf(str, "", 8); r = 0; return r; } static int ln_addField_XML(char *name, struct json_object *field, es_str_t **str) { int r; int i; const char *value; struct json_object *obj; assert(field != NULL); assert(str != NULL); assert(*str != NULL); CHKR(es_addBuf(str, "", 2)); switch(json_object_get_type(field)) { case json_type_array: for (i = json_object_array_length(field) - 1; i >= 0; i--) { CHKN(obj = json_object_array_get_idx(field, i)); CHKN(value = json_object_get_string(obj)); CHKR(ln_addValue_XML(value, str)); } break; case json_type_string: case json_type_int: CHKN(value = json_object_get_string(field)); CHKR(ln_addValue_XML(value, str)); break; case json_type_null: case json_type_boolean: case json_type_double: case json_type_object: CHKR(es_addBuf(str, "***unsupported type***", sizeof("***unsupported type***")-1)); break; default: CHKR(es_addBuf(str, "***OBJECT***", sizeof("***OBJECT***")-1)); } CHKR(es_addBuf(str, "", 8)); r = 0; done: return r; } static inline int ln_addTags_XML(struct json_object *taglist, es_str_t **str) { int r = 0; struct json_object *tagObj; const char *tagCstr; int i; CHKR(es_addBuf(str, "", 12)); for (i = json_object_array_length(taglist) - 1; i >= 0; i--) { CHKR(es_addBuf(str, "", 5)); CHKN(tagObj = json_object_array_get_idx(taglist, i)); CHKN(tagCstr = json_object_get_string(tagObj)); CHKR(es_addBuf(str, (char*)tagCstr, strlen(tagCstr))); CHKR(es_addBuf(str, "", 6)); } CHKR(es_addBuf(str, "", 13)); done: return r; } int ln_fmtEventToXML(struct json_object *json, es_str_t **str) { int r = -1; struct json_object *tags; assert(json != NULL); assert(json_object_is_type(json, json_type_object)); if((*str = es_newStr(256)) == NULL) goto done; es_addBuf(str, "", 7); if(json_object_object_get_ex(json, "event.tags", &tags)) { CHKR(ln_addTags_XML(tags, str)); } struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { char *const name = (char*) json_object_iter_peek_name(&it); if (strcmp(name, "event.tags")) { ln_addField_XML(name, json_object_iter_peek_value(&it), str); } json_object_iter_next(&it); } es_addBuf(str, "", 8); done: return r; } liblognorm-2.0.8/src/helpers.h000066400000000000000000000005111511425433100162630ustar00rootroot00000000000000/** * @file pdag.c * @brief Implementation of the parse dag object. * @class ln_pdag pdag.h *//* * Copyright 2015 by Rainer Gerhards and Adiscon GmbH. * * Released under ASL 2.0. */ #ifndef LIBLOGNORM_HELPERS_H #define LIBLOGNORM_HELPERS_H static inline int myisdigit(char c) { return (c >= '0' && c <= '9'); } #endif liblognorm-2.0.8/src/internal.h000066400000000000000000000074121511425433100164440ustar00rootroot00000000000000/** * @file internal.h * @brief Internal things just needed for building the library, but * not to be installed. *//** * @mainpage * Liblognorm is an easy to use and fast samples-based log normalization * library. * * It can be passed a stream of arbitrary log messages, one at a time, and for * each message it will output well-defined name-value pairs and a set of * tags describing the message. * * For further details, see it's initial announcement available at * https://rainer.gerhards.net/2010/10/introducing-liblognorm.html * * The public interface of this library is describe in liblognorm.h. * * Liblognorm fully supports Unicode. Like most Linux tools, it operates * on UTF-8 natively, called "passive mode". This was decided because we * so can keep the size of data structures small while still supporting * all of the world's languages (actually more than when we did UCS-2). * * At the technical level, we can handle UTF-8 multibyte sequences transparently. * Liblognorm needs to look at a few US-ASCII characters to do the * sample base parsing (things to indicate fields), so this is no * issue. Inside the parse tree, a multibyte sequence can simple be processed * as if it were a sequence of different characters that make up a their * own symbol each. In fact, this even allows for somewhat greater parsing * speed. *//* * * liblognorm - a fast samples-based log normalization library * Copyright 2010-2018 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef INTERNAL_H_INCLUDED #define INTERNAL_H_INCLUDED /* the jump-misses-init gcc warning is overdoing when we jump to the * exit of a function to get proper finalization. So let's disable it. * rgerhards, 2018-04-25 */ #pragma GCC diagnostic ignored "-Wjump-misses-init" #include "liblognorm.h" #include /* we need to turn off this warning, as it also comes up in C99 mode, which * we use. */ #ifndef _AIX #pragma GCC diagnostic ignored "-Wdeclaration-after-statement" #endif /* support for simple error checking */ #define CHKR(x) \ if((r = (x)) != 0) goto done #define CHKN(x) \ if((x) == NULL) { \ r = LN_NOMEM; \ goto done; \ } #define FAIL(e) {r = (e); goto done;} static inline char* ln_es_str2cstr(es_str_t **str) { int r = -1; char *buf; if (es_strlen(*str) == (*str)->lenBuf) { CHKR(es_extendBuf(str, 1)); } CHKN(buf = (char*)es_getBufAddr(*str)); buf[es_strlen(*str)] = '\0'; return buf; done: return NULL; } const char * ln_DataForDisplayCharTo(__attribute__((unused)) ln_ctx ctx, void *const pdata); const char * ln_DataForDisplayLiteral(__attribute__((unused)) ln_ctx ctx, void *const pdata); const char * ln_JsonConfLiteral(__attribute__((unused)) ln_ctx ctx, void *const pdata); /* here we add some stuff from the compatibility layer */ #ifndef HAVE_STRNDUP char * strndup(const char *s, size_t n); #endif #endif /* #ifndef INTERNAL_H_INCLUDED */ liblognorm-2.0.8/src/liblognorm.c000066400000000000000000000076041511425433100167720ustar00rootroot00000000000000/* This file implements the liblognorm API. * See header file for descriptions. * * liblognorm - a fast samples-based log normalization library * Copyright 2013 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include "liblognorm.h" #include "lognorm.h" #include "annot.h" #include "samp.h" #include "v1_liblognorm.h" #include "v1_ptree.h" #define CHECK_CTX \ if(ctx->objID != LN_ObjID_CTX) { \ r = -1; \ goto done; \ } const char * ln_version(void) { return VERSION; } int ln_hasAdvancedStats(void) { #ifdef ADVANCED_STATS return 1; #else return 0; #endif } ln_ctx ln_initCtx(void) { ln_ctx ctx; if((ctx = calloc(1, sizeof(struct ln_ctx_s))) == NULL) goto done; #ifdef HAVE_JSON_GLOBAL_SET_STRING_HASH json_global_set_string_hash(JSON_C_STR_HASH_PERLLIKE); #endif #ifdef HAVE_JSON_GLOBAL_SET_PRINTBUF_INITIAL_SIZE json_global_set_printbuf_initial_size(2048); #endif ctx->objID = LN_ObjID_CTX; ctx->dbgCB = NULL; ctx->opts = 0; /* we add an root for the empty word, this simplifies parse * dag handling. */ if((ctx->pdag = ln_newPDAG(ctx)) == NULL) { free(ctx); ctx = NULL; goto done; } /* same for annotation set */ if((ctx->pas = ln_newAnnotSet(ctx)) == NULL) { ln_pdagDelete(ctx->pdag); free(ctx); ctx = NULL; goto done; } done: return ctx; } void ln_setCtxOpts(ln_ctx ctx, const unsigned opts) { ctx->opts |= opts; } int ln_exitCtx(ln_ctx ctx) { int r = 0; CHECK_CTX; ln_dbgprintf(ctx, "exitCtx %p", ctx); ctx->objID = LN_ObjID_None; /* prevent double free */ /* support for old cruft */ if(ctx->ptree != NULL) ln_deletePTree(ctx->ptree); /* end support for old cruft */ if(ctx->pdag != NULL) ln_pdagDelete(ctx->pdag); for(int i = 0 ; i < ctx->nTypes ; ++i) { free((void*)ctx->type_pdags[i].name); ln_pdagDelete(ctx->type_pdags[i].pdag); } free(ctx->type_pdags); if(ctx->rulePrefix != NULL) es_deleteStr(ctx->rulePrefix); if(ctx->pas != NULL) ln_deleteAnnotSet(ctx->pas); free(ctx); done: return r; } int ln_setDebugCB(ln_ctx ctx, void (*cb)(void*, const char*, size_t), void *cookie) { int r = 0; CHECK_CTX; ctx->dbgCB = cb; ctx->dbgCookie = cookie; done: return r; } int ln_setErrMsgCB(ln_ctx ctx, void (*cb)(void*, const char*, size_t), void *cookie) { int r = 0; CHECK_CTX; ctx->errmsgCB = cb; ctx->errmsgCookie = cookie; done: return r; } int ln_loadSamples(ln_ctx ctx, const char *file) { int r = 0; const char *tofree; CHECK_CTX; ctx->conf_file = tofree = strdup(file); ctx->conf_ln_nbr = 0; ++ctx->include_level; r = ln_sampLoad(ctx, file); --ctx->include_level; free((void*)tofree); ctx->conf_file = NULL; done: return r; } int ln_loadSamplesFromString(ln_ctx ctx, const char *string) { int r = 0; const char *tofree; CHECK_CTX; ctx->conf_file = tofree = strdup("--NO-FILE--"); ctx->conf_ln_nbr = 0; ++ctx->include_level; r = ln_sampLoadFromString(ctx, string); --ctx->include_level; free((void*)tofree); ctx->conf_file = NULL; done: return r; } liblognorm-2.0.8/src/liblognorm.h000066400000000000000000000224151511425433100167740ustar00rootroot00000000000000/** * @file liblognorm.h * @brief The public liblognorm API. * * Functions other than those defined here MUST not be called by * a liblognorm "user" application. * * This file is meant to be included by applications using liblognorm. * For lognorm library files themselves, include "lognorm.h". *//** * @mainpage * Liblognorm is an easy to use and fast samples-based log normalization * library. * * It can be passed a stream of arbitrary log messages, one at a time, and for * each message it will output well-defined name-value pairs and a set of * tags describing the message. * * For further details, see it's initial announcement available at * https://rainer.gerhards.net/2010/10/introducing-liblognorm.html * * The public interface of this library is describe in liblognorm.h. * * Liblognorm fully supports Unicode. Like most Linux tools, it operates * on UTF-8 natively, called "passive mode". This was decided because we * so can keep the size of data structures small while still supporting * all of the world's languages (actually more than when we did UCS-2). * * At the technical level, we can handle UTF-8 multibyte sequences transparently. * Liblognorm needs to look at a few US-ASCII characters to do the * sample base parsing (things to indicate fields), so this is no * issue. Inside the parse tree, a multibyte sequence can simple be processed * as if it were a sequence of different characters that make up a their * own symbol each. In fact, this even allows for somewhat greater parsing * speed. *//* * * liblognorm - a fast samples-based log normalization library * Copyright 2010-2017 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef LIBLOGNORM_H_INCLUDED #define LIBLOGNORM_H_INCLUDED #include /* we need size_t */ #include /* error codes */ #define LN_NOMEM -1 #define LN_INVLDFDESCR -1 #define LN_BADCONFIG -250 #define LN_BADPARSERSTATE -500 #define LN_WRONGPARSER -1000 #define LN_RB_LINE_TOO_LONG -1001 #define LN_OVER_SIZE_LIMIT -1002 /** * The library context descriptor. * This is used to permit multiple independent instances of the * library to be called within a single program. This is most * useful for plugin-based architectures. */ typedef struct ln_ctx_s* ln_ctx; /* API */ /** * Return library version string. * * Returns the version of the currently used library. * * @return Zero-Terminated library version string. */ /* Note: this MUST NOT be inline to make sure the actual library * has the right version, not just what was used to compile! */ const char *ln_version(void); /** * Return if library is build with advanced statistics * activated. * * @return 1 if advanced stats are active, 0 if not */ int ln_hasAdvancedStats(void); /** * Initialize a library context. * * To prevent memory leaks, ln_exitCtx() must be called on a library * context that is no longer needed. * * @return new library context or NULL if an error occurred */ ln_ctx ln_initCtx(void); /** * Inherit control attributes from a library context. * * This does not copy the parse-tree, but does copy * behaviour-controlling attributes such as enableRegex. * * Just as with ln_initCtx, ln_exitCtx() must be called on a library * context that is no longer needed. * * @return new library context or NULL if an error occurred */ ln_ctx ln_inherittedCtx(ln_ctx parent); /** * Discard a library context. * * Free's the resources associated with the given library context. It * MUST NOT be accessed after calling this function. * * @param ctx The context to be discarded. * * @return Returns zero on success, something else otherwise. */ int ln_exitCtx(ln_ctx ctx); /* binary values, so that we can "or" them together */ #define LN_CTXOPT_ALLOW_REGEX 0x01 /**< permit regex matching */ #define LN_CTXOPT_ADD_EXEC_PATH 0x02 /**< add exec_path attribute (time-consuming!) */ #define LN_CTXOPT_ADD_ORIGINALMSG 0x04 /**< always add original message to output (not just in error case) */ #define LN_CTXOPT_ADD_RULE 0x08 /**< add mockup rule */ #define LN_CTXOPT_ADD_RULE_LOCATION 0x10 /**< add rule location (file, lineno) to metadata */ /** * Set options on ctx. * * @param ctx The context to be modified. * @param opts a potentially or-ed list of options, see LN_CTXOPT_* */ void ln_setCtxOpts(ln_ctx ctx, unsigned opts); /** * Set a debug message handler (callback). * * Liblognorm can provide helpful information for debugging * - it's internal processing * - the way a log message is being normalized * * It does so by emitting "interesting" information about its processing * at various stages. A caller can obtain this information by registering * an entry point. When done so, liblognorm will call the entry point * whenever it has something to emit. Note that debugging can be rather * verbose. * * The callback will be called with the following three parameters in that order: * - the caller-provided cookie * - a zero-terminated string buffer * - the length of the string buffer, without the trailing NUL byte * * @note * The provided callback function must not call any liblognorm * APIs except when specifically flagged as safe for calling by a debug * callback handler. * * @param[in] ctx The library context to apply callback to. * @param[in] cb The function to be called for debugging * @param[in] cookie Opaque cookie to be passed down to debug handler. Can be * used for some state tracking by the caller. This is defined as * void* to support pointers. To play it safe, a pointer should be * passed (but adventurous folks may also use an unsigned). * * @return Returns zero on success, something else otherwise. */ int ln_setDebugCB(ln_ctx ctx, void (*cb)(void*, const char*, size_t), void *cookie); /** * Set a error message handler (callback). * * If set, this is used to emit error messages of interest to the user, e.g. * on failures during rulebase load. It is suggested that a caller uses this * feedback to aid its users in resolving issues. * Its semantics are otherwise exactly the same like ln_setDebugCB(). */ int ln_setErrMsgCB(ln_ctx ctx, void (*cb)(void*, const char*, size_t), void *cookie); /** * enable or disable debug mode. * * @param[in] ctx context * @param[in] b boolean 0 - disable debug mode, 1 - enable debug mode */ void ln_enableDebug(ln_ctx ctx, int i); /** * Load a (log) sample file. * * The file must contain log samples in syntactically correct format. Samples are added * to set already loaded in the current context. If there is a sample with duplicate * semantics, this sample will be ignored. Most importantly, this can \b not be used * to change tag assignments for a given sample. * * @param[in] ctx The library context to apply callback to. * @param[in] file Name of file to be loaded. * * @return Returns zero on success, something else otherwise. */ int ln_loadSamples(ln_ctx ctx, const char *file); /** * Load a rulebase via a string. * * Note: this can only load v2 samples, v1 is NOT supported. * * @param[in] ctx The library context to apply callback to. * @param[in] string The string with the actual rulebase. * * @return Returns zero on success, something else otherwise. */ int ln_loadSamplesFromString(ln_ctx ctx, const char *string); /** * Normalize a message. * * This is the main library entry point. It is called with a message * to normalize and will return a normalized in-memory representation * of it. * * If an error occurs, the function returns -1. In that case, an * in-memory event representation has been generated if event is * non-NULL. In that case, the event contains further error details in * normalized form. * * @note * This function works on byte-counted strings and as such is able to * process NUL bytes if they occur inside the message. On the other hand, * this means the the correct messages size, \b excluding the NUL byte, * must be provided. * * @param[in] ctx The library context to use. * @param[in] str The message string (see note above). * @param[in] strLen The length of the message in bytes. * @param[out] json_p A new event record or NULL if an error occurred. Must be * destructed if no longer needed. * * @return Returns zero on success, something else otherwise. */ int ln_normalize(ln_ctx ctx, const char *str, const size_t strLen, struct json_object **json_p); #endif /* #ifndef LOGNORM_H_INCLUDED */ liblognorm-2.0.8/src/lognorm-features.h.in000066400000000000000000000000761511425433100205250ustar00rootroot00000000000000#if @FEATURE_REGEXP@ #define LOGNORM_REGEX_SUPPORTED 1 #endif liblognorm-2.0.8/src/lognorm.c000066400000000000000000000067331511425433100163050ustar00rootroot00000000000000/* liblognorm - a fast samples-based log normalization library * Copyright 2010 by Rainer Gerhards and Adiscon GmbH. * * This file is part of liblognorm. * * Released under ASL 2.0 */ #include "config.h" #include #include #include #include #include "liblognorm.h" #include "lognorm.h" /* Code taken from rsyslog ASL 2.0 code. * From varmojfekoj's mail on why he provided rs_strerror_r(): * There are two problems with strerror_r(): * I see you've rewritten some of the code which calls it to use only * the supplied buffer; unfortunately the GNU implementation sometimes * doesn't use the buffer at all and returns a pointer to some * immutable string instead, as noted in the man page. * * The other problem is that on some systems strerror_r() has a return * type of int. * * So I've written a wrapper function rs_strerror_r(), which should * take care of all this and be used instead. */ static char * rs_strerror_r(const int errnum, char *const buf, const size_t buflen) { #ifndef HAVE_STRERROR_R char *pszErr; pszErr = strerror(errnum); snprintf(buf, buflen, "%s", pszErr); #else # ifdef STRERROR_R_CHAR_P char *p = strerror_r(errnum, buf, buflen); if (p != buf) { strncpy(buf, p, buflen); buf[buflen - 1] = '\0'; } # else strerror_r(errnum, buf, buflen); # endif #endif return buf; } /** * Generate some debug message and call the caller provided callback. * * Will first check if a user callback is registered. If not, returns * immediately. */ void ln_dbgprintf(ln_ctx ctx, const char *fmt, ...) { va_list ap; char buf[8*1024]; size_t lenBuf; if(ctx->dbgCB == NULL) goto done; va_start(ap, fmt); lenBuf = vsnprintf(buf, sizeof(buf), fmt, ap); va_end(ap); if(lenBuf >= sizeof(buf)) { /* prevent buffer overruns and garbage display */ buf[sizeof(buf) - 5] = '.'; buf[sizeof(buf) - 4] = '.'; buf[sizeof(buf) - 3] = '.'; buf[sizeof(buf) - 2] = '\n'; buf[sizeof(buf) - 1] = '\0'; lenBuf = sizeof(buf) - 1; } ctx->dbgCB(ctx->dbgCookie, buf, lenBuf); done: return; } /** * Generate error message and call the caller provided callback. * eno is the OS errno. If non-zero, the OS error description * will be added after the user-provided string. * * Will first check if a user callback is registered. If not, returns * immediately. */ void ln_errprintf(const ln_ctx ctx, const int eno, const char *fmt, ...) { va_list ap; char buf[8*1024]; char errbuf[1024]; char finalbuf[9*1024]; size_t lenBuf; char *msg; if(ctx->errmsgCB == NULL) goto done; va_start(ap, fmt); lenBuf = vsnprintf(buf, sizeof(buf), fmt, ap); va_end(ap); if(lenBuf >= sizeof(buf)) { /* prevent buffer overruns and garbage display */ buf[sizeof(buf) - 5] = '.'; buf[sizeof(buf) - 4] = '.'; buf[sizeof(buf) - 3] = '.'; buf[sizeof(buf) - 2] = '\n'; buf[sizeof(buf) - 1] = '\0'; lenBuf = sizeof(buf) - 1; } if(eno != 0) { rs_strerror_r(eno, errbuf, sizeof(errbuf)-1); lenBuf = snprintf(finalbuf, sizeof(finalbuf), "%s: %s", buf, errbuf); msg = finalbuf; } else { msg = buf; } if(ctx->conf_file != NULL) { /* error during config processing, add line info */ const char *const m = strdup(msg); lenBuf = snprintf(finalbuf, sizeof(finalbuf), "rulebase file %s[%d]: %s", ctx->conf_file, ctx->conf_ln_nbr, m); msg = finalbuf; free((void*) m); } ctx->errmsgCB(ctx->dbgCookie, msg, lenBuf); ln_dbgprintf(ctx, "%s", msg); done: return; } void ln_enableDebug(ln_ctx ctx, int i) { ctx->debug = i & 0x01; } liblognorm-2.0.8/src/lognorm.h000066400000000000000000000066641511425433100163150ustar00rootroot00000000000000/** * @file lognorm.h * @brief Private data structures used by the liblognorm API. *//* * * liblognorm - a fast samples-based log normalization library * Copyright 2010 by Rainer Gerhards and Adiscon GmbH. * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef LIBLOGNORM_LOGNORM_HINCLUDED #define LIBLOGNORM_LOGNORM_HINCLUDED #include /* we need size_t */ #include "liblognorm.h" #include "pdag.h" #include "annot.h" /* some limits */ #define MAX_FIELDNAME_LEN 1024 #define MAX_TYPENAME_LEN 1024 #define LN_ObjID_None 0xFEFE0001 #define LN_ObjID_CTX 0xFEFE0001 struct ln_type_pdag { const char *name; ln_pdag *pdag; }; struct ln_ctx_s { unsigned objID; /**< a magic number to prevent some memory addressing errors */ void (*dbgCB)(void *cookie, const char *msg, size_t lenMsg); /**< user-provided debug output callback */ void *dbgCookie; /**< cookie to be passed to debug callback */ void (*errmsgCB)(void *cookie, const char *msg, size_t lenMsg); /**< user-provided error message callback */ void *errmsgCookie; /**< cookie to be passed to error message callback */ ln_pdag *pdag; /**< parse dag being used by this context */ ln_annotSet *pas; /**< associated set of annotations */ unsigned nNodes; /**< number of nodes in our parse tree */ unsigned char debug; /**< boolean: are we in debug mode? */ es_str_t *rulePrefix; /**< work variable for loading rule bases * this is the prefix string that will be prepended * to all rules before they are submitted to tree * building. */ unsigned opts; /**< specific options, see LN_CTXOPTS_* defines */ struct ln_type_pdag *type_pdags; /**< array of our type pdags */ int nTypes; /**< number of type pdags */ int version; /**< 1 or 2, depending on rulebase/algo version */ /* here follows stuff for the v1 subsystem -- do NOT make any changes * down here. This is strictly read-only. May also be removed some time in * the future. */ struct ln_ptree *ptree; /* end old cruft */ /* things for config processing / error message during it */ int include_level; /**< 1 for main rulebase file, higher for include levels */ const char *conf_file; /**< currently open config file or NULL, if none */ unsigned int conf_ln_nbr; /**< current config file line number */ }; void ln_dbgprintf(ln_ctx ctx, const char *fmt, ...) __attribute__((format(printf, 2, 3))); void ln_errprintf(ln_ctx ctx, const int eno, const char *fmt, ...) __attribute__((format(printf, 3, 4))); #define LN_DBGPRINTF(ctx, ...) if(ctx->dbgCB != NULL) { ln_dbgprintf(ctx, __VA_ARGS__); } //#define LN_DBGPRINTF(ctx, ...) #endif /* #ifndef LIBLOGNORM_LOGNORM_HINCLUDED */ liblognorm-2.0.8/src/lognormalizer.c000066400000000000000000000330701511425433100175060ustar00rootroot00000000000000/** * @file normalizer.c * @brief A small tool to normalize data. * * This is the most basic example demonstrating how to use liblognorm. * It loads log samples from the files specified on the command line, * reads to-be-normalized data from stdin and writes the normalized * form to stdout. Besides being an example, it also carries out useful * processing. * * @author Rainer Gerhards * *//* * liblognorm - a fast samples-based log normalization library * Copyright 2010-2016 by Rainer Gerhards and Adiscon GmbH. * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #ifdef _AIX #include #else #include #endif #include #include "liblognorm.h" #include "lognorm.h" #include "enc.h" /* we need to turn off this warning, as it also comes up in C99 mode, which * we use. */ #pragma GCC diagnostic ignored "-Wdeclaration-after-statement" static ln_ctx ctx; static int verbose = 0; #define OUTPUT_PARSED_RECS 0x01 #define OUTPUT_UNPARSED_RECS 0x02 static int recOutput = OUTPUT_PARSED_RECS | OUTPUT_UNPARSED_RECS; /**< controls which records to output */ static int outputSummaryLine = 0; static int outputNbrUnparsed = 0; static int addErrLineNbr = 0; /**< add line number info to unparsed events */ static int flatTags = 0; /**< print event.tags in JSON? */ static FILE *fpDOT; static es_str_t *encFmt = NULL; /**< a format string for encoder use */ static es_str_t *mandatoryTag = NULL; /**< tag which must be given so that mesg will be output. NULL=all */ static enum { f_syslog, f_json, f_xml, f_csv, f_raw } outfmt = f_json; static void errCallBack(void __attribute__((unused)) *cookie, const char *msg, size_t __attribute__((unused)) lenMsg) { fprintf(stderr, "liblognorm error: %s\n", msg); } static void dbgCallBack(void __attribute__((unused)) *cookie, const char *msg, size_t __attribute__((unused)) lenMsg) { fprintf(stderr, "liblognorm: %s\n", msg); } static void complain(const char *errmsg) { fprintf(stderr, "%s\n", errmsg); } /* rawmsg is, as the name says, the raw message, in case we have * "raw" formatter requested. */ static void outputEvent(struct json_object *json, const char *const rawmsg) { char *cstr = NULL; es_str_t *str = NULL; if(outfmt == f_raw) { printf("%s\n", rawmsg); return; } switch(outfmt) { case f_json: if(!flatTags) { json_object_object_del(json, "event.tags"); } cstr = (char*)json_object_to_json_string(json); break; case f_syslog: ln_fmtEventToRFC5424(json, &str); break; case f_xml: ln_fmtEventToXML(json, &str); break; case f_csv: ln_fmtEventToCSV(json, &str, encFmt); break; case f_raw: fprintf(stderr, "program error: f_raw should not occur " "here (file %s, line %d)\n", __FILE__, __LINE__); abort(); break; default: fprintf(stderr, "program error: default case should not occur " "here (file %s, line %d)\n", __FILE__, __LINE__); abort(); break; } if (str != NULL) cstr = es_str2cstr(str, NULL); if(verbose > 0) fprintf(stderr, "normalized: '%s'\n", cstr); printf("%s\n", cstr); if (str != NULL) free(cstr); es_deleteStr(str); } /* test if the tag exists */ static int eventHasTag(struct json_object *json, const char *tag) { struct json_object *tagbucket, *tagObj; int i; const char *tagCstr; if (tag == NULL) return 1; if (json_object_object_get_ex(json, "event.tags", &tagbucket)) { if (json_object_get_type(tagbucket) == json_type_array) { for (i = json_object_array_length(tagbucket) - 1; i >= 0; i--) { tagObj = json_object_array_get_idx(tagbucket, i); tagCstr = json_object_get_string(tagObj); if (!strcmp(tag, tagCstr)) return 1; } } } if (verbose > 1) printf("Mandatory tag '%s' has not been found\n", tag); return 0; } static void amendLineNbr(json_object *const json, const int line_nbr) { if(addErrLineNbr) { struct json_object *jval; jval = json_object_new_int(line_nbr); json_object_object_add(json, "lognormalizer.line_nbr", jval); } } #define DEFAULT_LINE_SIZE (10 * 1024) static char * read_line(FILE *fp) { size_t line_capacity = DEFAULT_LINE_SIZE; char *line = NULL; size_t line_len = 0; int ch = 0; do { ch = fgetc(fp); if (ch == EOF) break; if (line == NULL) { line = malloc(line_capacity); } else if (line_len == line_capacity) { line_capacity *= 2; line = realloc(line, line_capacity); } if (line == NULL) { fprintf(stderr, "Couldn't allocate working-buffer for log-line\n"); return NULL; } line[line_len++] = ch; } while(ch != '\n'); if (line != NULL) { line[--line_len] = '\0'; if(line_len > 0 && line[line_len - 1] == '\r') line[--line_len] = '\0'; } return line; } /* normalize input data */ static void normalize(void) { FILE *fp = stdin; char *line = NULL; struct json_object *json = NULL; long long unsigned numParsed = 0; long long unsigned numUnparsed = 0; long long unsigned numWrongTag = 0; char *mandatoryTagCstr = NULL; int line_nbr = 0; /* must be int to keep compatible with older json-c */ if (mandatoryTag != NULL) { mandatoryTagCstr = es_str2cstr(mandatoryTag, NULL); } while((line = read_line(fp)) != NULL) { ++line_nbr; if(verbose > 0) fprintf(stderr, "To normalize: '%s'\n", line); ln_normalize(ctx, line, strlen(line), &json); if(json != NULL) { if(eventHasTag(json, mandatoryTagCstr)) { struct json_object *dummy; const int parsed = !json_object_object_get_ex(json, "unparsed-data", &dummy); if(parsed) { numParsed++; if(recOutput & OUTPUT_PARSED_RECS) { outputEvent(json, line); } } else { numUnparsed++; amendLineNbr(json, line_nbr); if(recOutput & OUTPUT_UNPARSED_RECS) { outputEvent(json, line); } } } else { numWrongTag++; } json_object_put(json); json = NULL; } free(line); } if(outputNbrUnparsed && numUnparsed > 0) fprintf(stderr, "%llu unparsable entries\n", numUnparsed); if(numWrongTag > 0) fprintf(stderr, "%llu entries with wrong tag dropped\n", numWrongTag); if(outputSummaryLine) { fprintf(stderr, "%llu records processed, %llu parsed, %llu unparsed\n", numParsed+numUnparsed, numParsed, numUnparsed); } free(mandatoryTagCstr); } /** * Generate a command file for the GNU DOT tools. */ static void genDOT(void) { es_str_t *str; str = es_newStr(1024); ln_genDotPDAGGraph(ctx->pdag, &str); fwrite(es_getBufAddr(str), 1, es_strlen(str), fpDOT); } static void printVersion(void) { fprintf(stderr, "lognormalizer version: " VERSION "\n"); fprintf(stderr, "liblognorm version: %s\n", ln_version()); fprintf(stderr, "\tadvanced stats: %s\n", ln_hasAdvancedStats() ? "available" : "not available"); } static void handle_generic_option(const char* opt) { if (strcmp("allowRegex", opt) == 0) { ln_setCtxOpts(ctx, LN_CTXOPT_ALLOW_REGEX); } else if (strcmp("addExecPath", opt) == 0) { ln_setCtxOpts(ctx, LN_CTXOPT_ADD_EXEC_PATH); } else if (strcmp("addOriginalMsg", opt) == 0) { ln_setCtxOpts(ctx, LN_CTXOPT_ADD_ORIGINALMSG); } else if (strcmp("addRule", opt) == 0) { ln_setCtxOpts(ctx, LN_CTXOPT_ADD_RULE); } else if (strcmp("addRuleLocation", opt) == 0) { ln_setCtxOpts(ctx, LN_CTXOPT_ADD_RULE_LOCATION); } else { fprintf(stderr, "invalid -o option '%s'\n", opt); exit(1); } } static void usage(void) { fprintf(stderr, "Options:\n" " -r Rulebase to use. This is required option\n" " -H print summary line (nbr of msgs Handled)\n" " -U print number of unparsed messages (only if non-zero)\n" " -e\n" " Change output format. By default, json is used\n" " Raw is exactly like the input. It is useful in combination\n" " with -p/-P options to extract known good/bad messages\n" " -E Encoder-specific format (used for CSV, read docs)\n" " -T Include 'event.tags' in JSON format\n" " -oallowRegex Allow regexp matching (read docs about performance penalty)\n" " -oaddRule Add a mockup of the matching rule.\n" " -oaddRuleLocation Add location of matching rule to metadata\n" " -oaddExecPath Add exec_path attribute to output\n" " -oaddOriginalMsg Always add original message to output, not just in error case\n" " -p Print back only if the message has been parsed successfully\n" " -P Print back only if the message has NOT been parsed successfully\n" " -L Add source file line number information to unparsed line output\n" " -t Print back only messages matching the tag\n" " -v Print debug. When used 3 times, prints parse DAG\n" " -V Print version information\n" " -d Print DOT file to stdout and exit\n" " -d Save DOT file to the filename\n" " -s Print parse dag statistics and exit\n" " -S Print extended parse dag statistics and exit (includes -s)\n" " -x Print statistics as dot file (called only)\n" "\n" ); } int main(int argc, char *argv[]) { int opt; char *repository = NULL; int usedRB = 0; /* 0=no rule; 1=rule from rulebase; 2=rule from string */ int ret = 0; FILE *fpStats = NULL; FILE *fpStatsDOT = NULL; int extendedStats = 0; if((ctx = ln_initCtx()) == NULL) { complain("Could not initialize liblognorm context"); ret = 1; goto exit; } while((opt = getopt(argc, argv, "d:s:S:e:r:R:E:vVpPt:To:hHULx:")) != -1) { switch (opt) { case 'V': printVersion(); exit(1); break; case 'd': /* generate DOT file */ if(!strcmp(optarg, "")) { fpDOT = stdout; } else { if((fpDOT = fopen(optarg, "w")) == NULL) { perror(optarg); complain("Cannot open DOT file"); ret = 1; goto exit; } } break; case 'x': /* generate statistics DOT file */ if(!strcmp(optarg, "")) { fpStatsDOT = stdout; } else { if((fpStatsDOT = fopen(optarg, "w")) == NULL) { perror(optarg); complain("Cannot open statistics DOT file"); ret = 1; goto exit; } } break; case 'S': /* generate pdag statistic file */ extendedStats = 1; /* INTENTIONALLY NO BREAK! - KEEP order! */ /*FALLTHROUGH*/ case 's': /* generate pdag statistic file */ if(!strcmp(optarg, "-")) { fpStats = stdout; } else { if((fpStats = fopen(optarg, "w")) == NULL) { perror(optarg); complain("Cannot open parser statistics file"); ret = 1; goto exit; } } break; case 'v': verbose++; break; case 'E': /* encoder-specific format string (will be validated by encoder) */ encFmt = es_newStrFromCStr(optarg, strlen(optarg)); break; case 'p': recOutput = OUTPUT_PARSED_RECS; break; case 'P': recOutput = OUTPUT_UNPARSED_RECS; break; case 'H': outputSummaryLine = 1; break; case 'U': outputNbrUnparsed = 1; break; case 'L': addErrLineNbr = 1; break; case 'T': flatTags = 1; break; case 'e': /* encoder to use */ if(!strcmp(optarg, "json")) { outfmt = f_json; } else if(!strcmp(optarg, "xml")) { outfmt = f_xml; } else if(!strcmp(optarg, "cee-syslog")) { outfmt = f_syslog; } else if(!strcmp(optarg, "csv")) { outfmt = f_csv; } else if(!strcmp(optarg, "raw")) { outfmt = f_raw; } break; case 'r': /* rule base to use */ if(usedRB != 2) { repository = optarg; usedRB = 1; } else { usedRB = -1; } break; case 'R': if(usedRB != 1) { repository = optarg; usedRB = 2; } else { usedRB = -1; } break; case 't': /* if given, only messages tagged with the argument are output */ mandatoryTag = es_newStrFromCStr(optarg, strlen(optarg)); break; case 'o': handle_generic_option(optarg); break; case 'h': default: usage(); ret = 1; goto exit; break; } } if(repository == NULL) { complain("Samples repository or String must be given (-r or -R)"); ret = 1; goto exit; } if(usedRB == -1) { complain("Only use one rulebase (-r or -R)"); ret = 1; goto exit; } ln_setErrMsgCB(ctx, errCallBack, NULL); if(verbose) { ln_setDebugCB(ctx, dbgCallBack, NULL); ln_enableDebug(ctx, 1); } if(usedRB == 1) { if(ln_loadSamples(ctx, repository)) { fprintf(stderr, "fatal error: cannot load rulebase\n"); exit(1); } } else if(usedRB == 2) { if(ln_loadSamplesFromString(ctx, repository)) { fprintf(stderr, "fatal error: cannot load rule from String\n"); exit(1); } } if(verbose > 0) fprintf(stderr, "number of tree nodes: %d\n", ctx->nNodes); if(fpDOT != NULL) { genDOT(); ret=1; goto exit; } if(verbose > 2) ln_displayPDAG(ctx); normalize(); if(fpStats != NULL) { ln_fullPdagStats(ctx, fpStats, extendedStats); } if(fpStatsDOT != NULL) { ln_fullPDagStatsDOT(ctx, fpStatsDOT); } exit: if (ctx) ln_exitCtx(ctx); if (encFmt != NULL) free(encFmt); return ret; } liblognorm-2.0.8/src/parser.c000066400000000000000000002737251511425433100161330ustar00rootroot00000000000000/* * liblognorm - a fast samples-based log normalization library * Copyright 2010-2018 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include #include #include #include #include #include #include #include #include #include "liblognorm.h" #include "lognorm.h" #include "internal.h" #include "parser.h" #include "samp.h" #include "helpers.h" #ifdef FEATURE_REGEXP #include #include #endif /* how should output values be formatted? */ enum FMT_MODE { FMT_AS_STRING = 0, FMT_AS_NUMBER = 1, FMT_AS_TIMESTAMP_UX = 2, FMT_AS_TIMESTAMP_UX_MS = 3 }; /* some helpers */ static inline int hParseInt(const unsigned char **buf, size_t *lenBuf) { const unsigned char *p = *buf; size_t len = *lenBuf; int i = 0; while(len > 0 && myisdigit(*p)) { i = i * 10 + *p - '0'; ++p; --len; } *buf = p; *lenBuf = len; return i; } /* parser _parse interface * * All parsers receive * * @param[in] npb->str the to-be-parsed string * @param[in] npb->strLen length of the to-be-parsed string * @param[in] offs an offset into the string * @param[in] pointer to parser data block * @param[out] parsed bytes * @param[out] value ptr to json object containing parsed data * (can be unused, but if used *value MUST be NULL on entry) * * They will try to parse out "their" object from the string. If they * succeed, they: * * return 0 on success and LN_WRONGPARSER if this parser could * not successfully parse (but all went well otherwise) and something * else in case of an error. */ #define PARSER_Parse(ParserName) \ int ln_v2_parse##ParserName( \ npb_t *const npb, \ size_t *const offs, \ __attribute__((unused)) void *const pdata, \ size_t *parsed, \ struct json_object **value) \ { \ int r = LN_WRONGPARSER; \ *parsed = 0; #define FAILParser \ goto parserdone; /* suppress warnings */ \ parserdone: \ r = 0; \ goto done; /* suppress warnings */ \ done: #define ENDFailParser \ return r; \ } /* Return printable representation of parser content for * display purposes. This must not be 100% exact, but provide * a good indication of what it contains for a human. * @param[data] data parser data block * @return pointer to c string, NOT to be freed */ #define PARSER_DataForDisplay(ParserName) \ const char * ln_DataForDisplay##ParserName(__attribute__((unused)) ln_ctx ctx, void *const pdata) /* Return JSON parser config. This is primarily for comparison * of parser equalness. * @param[data] data parser data block * @return pointer to c string, NOT to be freed */ #define PARSER_JsonConf(ParserName) \ const char * ln_JsonConf##ParserName(__attribute__((unused)) ln_ctx ctx, void *const pdata) /* parser constructor * @param[in] json config json items * @param[out] data parser data block (to be allocated) * At minimum, *data must be set to NULL * @return error status (0 == OK) */ #define PARSER_Construct(ParserName) \ int ln_construct##ParserName( \ __attribute__((unused)) ln_ctx ctx, \ __attribute__((unused)) json_object *const json, \ void **pdata) /* parser destructor * @param[data] data parser data block (to be de-allocated) */ #define PARSER_Destruct(ParserName) \ void ln_destruct##ParserName(__attribute__((unused)) ln_ctx ctx, void *const pdata) /* the following table saves us from computing an additional date to get * the ordinal day of the year - at least from 1967-2099 * Note: non-2038+ compliant systems (Solaris) will generate compiler * warnings on the post 2038-rollover years. */ static const int yearInSec_startYear = 1967; /* for x in $(seq 1967 2099) ; do * printf %s', ' $(date --date="Dec 31 ${x} UTC 23:59:59" +%s) * done |fold -w 70 -s */ static const time_t yearInSecs[] = { -63158401, -31536001, -1, 31535999, 63071999, 94694399, 126230399, 157766399, 189302399, 220924799, 252460799, 283996799, 315532799, 347155199, 378691199, 410227199, 441763199, 473385599, 504921599, 536457599, 567993599, 599615999, 631151999, 662687999, 694223999, 725846399, 757382399, 788918399, 820454399, 852076799, 883612799, 915148799, 946684799, 978307199, 1009843199, 1041379199, 1072915199, 1104537599, 1136073599, 1167609599, 1199145599, 1230767999, 1262303999, 1293839999, 1325375999, 1356998399, 1388534399, 1420070399, 1451606399, 1483228799, 1514764799, 1546300799, 1577836799, 1609459199, 1640995199, 1672531199, 1704067199, 1735689599, 1767225599, 1798761599, 1830297599, 1861919999, 1893455999, 1924991999, 1956527999, 1988150399, 2019686399, 2051222399, 2082758399, 2114380799, 2145916799, 2177452799, 2208988799, 2240611199, 2272147199, 2303683199, 2335219199, 2366841599, 2398377599, 2429913599, 2461449599, 2493071999, 2524607999, 2556143999, 2587679999, 2619302399, 2650838399, 2682374399, 2713910399, 2745532799, 2777068799, 2808604799, 2840140799, 2871763199, 2903299199, 2934835199, 2966371199, 2997993599, 3029529599, 3061065599, 3092601599, 3124223999, 3155759999, 3187295999, 3218831999, 3250454399, 3281990399, 3313526399, 3345062399, 3376684799, 3408220799, 3439756799, 3471292799, 3502915199, 3534451199, 3565987199, 3597523199, 3629145599, 3660681599, 3692217599, 3723753599, 3755375999, 3786911999, 3818447999, 3849983999, 3881606399, 3913142399, 3944678399, 3976214399, 4007836799, 4039372799, 4070908799, 4102444799}; /** * convert syslog timestamp to time_t * Note: it would be better to use something similar to mktime() here. * Unfortunately, mktime() semantics are problematic: first of all, it * works on local time, on the machine's time zone. In syslog, we have * to deal with multiple time zones at once, so we cannot plainly rely * on the local zone, and so we cannot rely on mktime(). One solution would * be to refactor all time-related functions so that they are all guarded * by a mutex to ensure TZ consistency (which would also enable us to * change the TZ at will for specific function calls). But that would * potentially mean a lot of overhead. * Also, mktime() has some side effects, at least setting of tzname. With * a refactoring as described above that should probably not be a problem, * but would also need more work. For some more thoughts on this topic, * have a look here: * http://stackoverflow.com/questions/18355101/is-standard-c-mktime-thread-safe-on-linux * In conclusion, we keep our own code for generating the unix timestamp. * rgerhards, 2016-03-02 (taken from rsyslog sources) */ static time_t syslogTime2time_t(const int year, const int month, const int day, const int hour, const int minute, const int second, const int OffsetHour, const int OffsetMinute, const char OffsetMode) { long MonthInDays, NumberOfYears, NumberOfDays; int utcOffset; time_t TimeInUnixFormat; if(year < 1970 || year > 2100) { TimeInUnixFormat = 0; goto done; } /* Counting how many Days have passed since the 01.01 of the * selected Year (Month level), according to the selected Month*/ switch(month) { case 1: MonthInDays = 0; //until 01 of January break; case 2: MonthInDays = 31; //until 01 of February - leap year handling down below! break; case 3: MonthInDays = 59; //until 01 of March break; case 4: MonthInDays = 90; //until 01 of April break; case 5: MonthInDays = 120; //until 01 of Mai break; case 6: MonthInDays = 151; //until 01 of June break; case 7: MonthInDays = 181; //until 01 of July break; case 8: MonthInDays = 212; //until 01 of August break; case 9: MonthInDays = 243; //until 01 of September break; case 10: MonthInDays = 273; //until 01 of Oktober break; case 11: MonthInDays = 304; //until 01 of November break; case 12: MonthInDays = 334; //until 01 of December break; default: /* this cannot happen (and would be a program error) * but we need the code to keep the compiler silent. */ MonthInDays = 0; /* any value fits ;) */ break; } /* adjust for leap years */ if((year % 100 != 0 && year % 4 == 0) || (year == 2000)) { if(month > 2) MonthInDays++; } /* 1) Counting how many Years have passed since 1970 2) Counting how many Days have passed since the 01.01 of the selected Year (Day level) according to the Selected Month and Day. Last day doesn't count, it should be until last day 3) Calculating this period (NumberOfDays) in seconds*/ NumberOfYears = year - yearInSec_startYear - 1; NumberOfDays = MonthInDays + day - 1; TimeInUnixFormat = (yearInSecs[NumberOfYears] + 1) + NumberOfDays * 86400; /*Add Hours, minutes and seconds */ TimeInUnixFormat += hour*60*60; TimeInUnixFormat += minute*60; TimeInUnixFormat += second; /* do UTC offset */ utcOffset = OffsetHour*3600 + OffsetMinute*60; if(OffsetMode == '+') utcOffset *= -1; /* if timestamp is ahead, we need to "go back" to UTC */ TimeInUnixFormat += utcOffset; done: return TimeInUnixFormat; } struct data_RFC5424Date { enum FMT_MODE fmt_mode; }; /** * Parse a TIMESTAMP as specified in RFC5424 (subset of RFC3339). */ PARSER_Parse(RFC5424Date) const unsigned char *pszTS; struct data_RFC5424Date *const data = (struct data_RFC5424Date*) pdata; /* variables to temporarily hold time information while we parse */ int year; int month; int day; int hour; /* 24 hour clock */ int minute; int second; int secfrac; /* fractional seconds (must be 32 bit!) */ int secfracPrecision; int OffsetHour; /* UTC offset in hours */ int OffsetMinute; /* UTC offset in minutes */ char OffsetMode; size_t len; size_t orglen; /* end variables to temporarily hold time information while we parse */ pszTS = (unsigned char*) npb->str + *offs; len = orglen = npb->strLen - *offs; year = hParseInt(&pszTS, &len); /* We take the liberty to accept slightly malformed timestamps e.g. in * the format of 2003-9-1T1:0:0. */ if(len == 0 || *pszTS++ != '-') goto done; --len; month = hParseInt(&pszTS, &len); if(month < 1 || month > 12) goto done; if(len == 0 || *pszTS++ != '-') goto done; --len; day = hParseInt(&pszTS, &len); if(day < 1 || day > 31) goto done; if(len == 0 || *pszTS++ != 'T') goto done; --len; hour = hParseInt(&pszTS, &len); if(hour < 0 || hour > 23) goto done; if(len == 0 || *pszTS++ != ':') goto done; --len; minute = hParseInt(&pszTS, &len); if(minute < 0 || minute > 59) goto done; if(len == 0 || *pszTS++ != ':') goto done; --len; second = hParseInt(&pszTS, &len); if(second < 0 || second > 60) goto done; /* Now let's see if we have secfrac */ if(len > 0 && *pszTS == '.') { --len; const unsigned char *pszStart = ++pszTS; secfrac = hParseInt(&pszTS, &len); secfracPrecision = (int) (pszTS - pszStart); } else { secfracPrecision = 0; secfrac = 0; } /* check the timezone */ if(len == 0) goto done; if(*pszTS == 'Z') { OffsetHour = 0; OffsetMinute = 0; OffsetMode = '+'; --len; pszTS++; /* eat Z */ } else if((*pszTS == '+') || (*pszTS == '-')) { OffsetMode = *pszTS; --len; pszTS++; OffsetHour = hParseInt(&pszTS, &len); if(OffsetHour < 0 || OffsetHour > 23) goto done; if(len == 0 || *pszTS++ != ':') goto done; --len; OffsetMinute = hParseInt(&pszTS, &len); if(OffsetMinute < 0 || OffsetMinute > 59) goto done; } else { /* there MUST be TZ information */ goto done; } if(len > 0) { if(*pszTS != ' ') /* if it is not a space, it can not be a "good" time */ goto done; } /* we had success, so update parse pointer */ *parsed = orglen - len; if(value != NULL) { if(data->fmt_mode == FMT_AS_STRING) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } else { int64_t timestamp = syslogTime2time_t(year, month, day, hour, minute, second, OffsetHour, OffsetMinute, OffsetMode); if(data->fmt_mode == FMT_AS_TIMESTAMP_UX_MS) { timestamp *= 1000; /* simulate pow(), do not use math lib! */ int div = 1; if(secfracPrecision == 1) { secfrac *= 100; } else if(secfracPrecision == 2) { secfrac *= 10; } else if(secfracPrecision > 3) { for(int i = 0 ; i < (secfracPrecision - 3) ; ++i) div *= 10; } timestamp += secfrac / div; } *value = json_object_new_int64(timestamp); } } r = 0; /* success */ done: return r; } PARSER_Construct(RFC5424Date) { int r = 0; struct data_RFC5424Date *data = (struct data_RFC5424Date*) calloc(1, sizeof(struct data_RFC5424Date)); data->fmt_mode = FMT_AS_STRING; if(json == NULL) goto done; struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(!strcmp(key, "format")) { const char *fmtmode = json_object_get_string(val); if(!strcmp(fmtmode, "timestamp-unix")) { data->fmt_mode = FMT_AS_TIMESTAMP_UX; } else if(!strcmp(fmtmode, "timestamp-unix-ms")) { data->fmt_mode = FMT_AS_TIMESTAMP_UX_MS; } else if(!strcmp(fmtmode, "string")) { data->fmt_mode = FMT_AS_STRING; } else { ln_errprintf(ctx, 0, "invalid value for date-rfc5424:format %s", fmtmode); } } else { if(!(strcmp(key, "name") == 0 && strcmp(json_object_get_string(val), "-") == 0)) { ln_errprintf(ctx, 0, "invalid param for date-rfc5424 %s", key); } } json_object_iter_next(&it); } done: *pdata = data; return r; } PARSER_Destruct(RFC5424Date) { free(pdata); } struct data_RFC3164Date { enum FMT_MODE fmt_mode; }; /** * Parse a RFC3164 Date. */ PARSER_Parse(RFC3164Date) const unsigned char *p; size_t len, orglen; struct data_RFC3164Date *const data = (struct data_RFC3164Date*) pdata; /* variables to temporarily hold time information while we parse */ int year; int month; int day; int hour; /* 24 hour clock */ int minute; int second; p = (unsigned char*) npb->str + *offs; orglen = len = npb->strLen - *offs; /* If we look at the month (Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec), * we may see the following character sequences occur: * * J(an/u(n/l)), Feb, Ma(r/y), A(pr/ug), Sep, Oct, Nov, Dec * * We will use this for parsing, as it probably is the * fastest way to parse it. */ if(len < 3) goto done; switch(*p++) { case 'j': case 'J': if(*p == 'a' || *p == 'A') { ++p; if(*p == 'n' || *p == 'N') { ++p; month = 1; } else goto done; } else if(*p == 'u' || *p == 'U') { ++p; if(*p == 'n' || *p == 'N') { ++p; month = 6; } else if(*p == 'l' || *p == 'L') { ++p; month = 7; } else goto done; } else goto done; break; case 'f': case 'F': if(*p == 'e' || *p == 'E') { ++p; if(*p == 'b' || *p == 'B') { ++p; month = 2; } else goto done; } else goto done; break; case 'm': case 'M': if(*p == 'a' || *p == 'A') { ++p; if(*p == 'r' || *p == 'R') { ++p; month = 3; } else if(*p == 'y' || *p == 'Y') { ++p; month = 5; } else goto done; } else goto done; break; case 'a': case 'A': if(*p == 'p' || *p == 'P') { ++p; if(*p == 'r' || *p == 'R') { ++p; month = 4; } else goto done; } else if(*p == 'u' || *p == 'U') { ++p; if(*p == 'g' || *p == 'G') { ++p; month = 8; } else goto done; } else goto done; break; case 's': case 'S': if(*p == 'e' || *p == 'E') { ++p; if(*p == 'p' || *p == 'P') { ++p; month = 9; } else goto done; } else goto done; break; case 'o': case 'O': if(*p == 'c' || *p == 'C') { ++p; if(*p == 't' || *p == 'T') { ++p; month = 10; } else goto done; } else goto done; break; case 'n': case 'N': if(*p == 'o' || *p == 'O') { ++p; if(*p == 'v' || *p == 'V') { ++p; month = 11; } else goto done; } else goto done; break; case 'd': case 'D': if(*p == 'e' || *p == 'E') { ++p; if(*p == 'c' || *p == 'C') { ++p; month = 12; } else goto done; } else goto done; break; default: goto done; } len -= 3; /* done month */ if(len == 0 || *p++ != ' ') goto done; --len; /* we accept a slightly malformed timestamp with one-digit days. */ if(*p == ' ') { --len; ++p; } day = hParseInt(&p, &len); if(day < 1 || day > 31) goto done; if(len == 0 || *p++ != ' ') goto done; --len; /* time part */ hour = hParseInt(&p, &len); if(hour > 1970 && hour < 2100) { /* if so, we assume this actually is a year. This is a format found * e.g. in Cisco devices. * year = hour; */ /* re-query the hour, this time it must be valid */ if(len == 0 || *p++ != ' ') goto done; --len; hour = hParseInt(&p, &len); } if(hour < 0 || hour > 23) goto done; if(len == 0 || *p++ != ':') goto done; --len; minute = hParseInt(&p, &len); if(minute < 0 || minute > 59) goto done; if(len == 0 || *p++ != ':') goto done; --len; second = hParseInt(&p, &len); if(second < 0 || second > 60) goto done; /* we provide support for an extra ":" after the date. While this is an * invalid format, it occurs frequently enough (e.g. with Cisco devices) * to permit it as a valid case. -- rgerhards, 2008-09-12 */ if(len > 0 && *p == ':') { ++p; /* just skip past it */ --len; } /* we had success, so update parse pointer */ *parsed = orglen - len; if(value != NULL) { if(data->fmt_mode == FMT_AS_STRING) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } else { /* we assume year == current year, so let's obtain current year */ struct tm tm; const time_t curr = time(NULL); gmtime_r(&curr, &tm); year = tm.tm_year + 1900; int64_t timestamp = syslogTime2time_t(year, month, day, hour, minute, second, 0, 0, '+'); if(data->fmt_mode == FMT_AS_TIMESTAMP_UX_MS) { /* we do not have more precise info, just bring * into common format! */ timestamp *= 1000; } *value = json_object_new_int64(timestamp); } } r = 0; /* success */ done: return r; } PARSER_Construct(RFC3164Date) { int r = 0; struct data_RFC3164Date *data = (struct data_RFC3164Date*) calloc(1, sizeof(struct data_RFC3164Date)); data->fmt_mode = FMT_AS_STRING; if(json == NULL) goto done; struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(!strcmp(key, "format")) { const char *fmtmode = json_object_get_string(val); if(!strcmp(fmtmode, "timestamp-unix")) { data->fmt_mode = FMT_AS_TIMESTAMP_UX; } else if(!strcmp(fmtmode, "timestamp-unix-ms")) { data->fmt_mode = FMT_AS_TIMESTAMP_UX_MS; } else if(!strcmp(fmtmode, "string")) { data->fmt_mode = FMT_AS_STRING; } else { ln_errprintf(ctx, 0, "invalid value for date-rfc3164:format %s", fmtmode); } } else { if(!(strcmp(key, "name") == 0 && strcmp(json_object_get_string(val), "-") == 0)) { ln_errprintf(ctx, 0, "invalid param for date-rfc3164 %s", key); } } json_object_iter_next(&it); } done: *pdata = data; return r; } PARSER_Destruct(RFC3164Date) { free(pdata); } struct data_Number { int64_t maxval; enum FMT_MODE fmt_mode; }; /** * Parse a Number. * Note that a number is an abstracted concept. We always represent it * as 64 bits (but may later change our mind if performance dictates so). */ PARSER_Parse(Number) const char *c; size_t i; int64_t val = 0; struct data_Number *const data = (struct data_Number*) pdata; enum FMT_MODE fmt_mode = FMT_AS_STRING; int64_t maxval = 0; if(data != NULL) { fmt_mode = data->fmt_mode; maxval = data->maxval; } assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; for (i = *offs; i < npb->strLen && myisdigit(c[i]); i++) val = val * 10 + c[i] - '0'; if(maxval > 0 && val > maxval) { LN_DBGPRINTF(npb->ctx, "number parser: val too large (max %" PRIu64 ", actual %" PRIu64 ")", maxval, val); goto done; } if (i == *offs) goto done; /* success, persist */ *parsed = i - *offs; if(value != NULL) { if(fmt_mode == FMT_AS_STRING) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } else { *value = json_object_new_int64(val); } } r = 0; /* success */ done: return r; } PARSER_Construct(Number) { int r = 0; struct data_Number *data = (struct data_Number*) calloc(1, sizeof(struct data_Number)); data->fmt_mode = FMT_AS_STRING; if(json == NULL) goto done; struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(!strcmp(key, "maxval")) { errno = 0; data->maxval = json_object_get_int64(val); if(errno != 0) { ln_errprintf(ctx, errno, "param 'maxval' must be integer but is: %s", json_object_to_json_string(val)); } } else if(!strcmp(key, "format")) { const char *fmtmode = json_object_get_string(val); if(!strcmp(fmtmode, "number")) { data->fmt_mode = FMT_AS_NUMBER; } else if(!strcmp(fmtmode, "string")) { data->fmt_mode = FMT_AS_STRING; } else { ln_errprintf(ctx, 0, "invalid value for number:format %s", fmtmode); } } else { if(!(strcmp(key, "name") == 0 && strcmp(json_object_get_string(val), "-") == 0)) { ln_errprintf(ctx, 0, "invalid param for number: %s", key); } } json_object_iter_next(&it); } done: *pdata = data; return r; } PARSER_Destruct(Number) { free(pdata); } struct data_Float { enum FMT_MODE fmt_mode; }; /** * Parse a Real-number in floating-pt form. */ PARSER_Parse(Float) const char *c; size_t i; const struct data_Float *const data = (struct data_Float*) pdata; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; int isNeg = 0; double val = 0; int seen_point = 0; double frac = 10; i = *offs; if (c[i] == '-') { isNeg = 1; i++; } for (; i < npb->strLen; i++) { if (c[i] == '.') { if (seen_point != 0) break; seen_point = 1; } else if (myisdigit(c[i])) { if(seen_point) { val += (c[i] - '0') / frac; frac *= 10; } else { val = val * 10 + c[i] - '0'; } } else { break; } } if (i == *offs) goto done; if(isNeg) val *= -1; /* success, persist */ *parsed = i - *offs; if(value != NULL) { if(data->fmt_mode == FMT_AS_STRING) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } else { char *serialized = strndup(npb->str+(*offs), *parsed); *value = json_object_new_double_s(val, serialized); free(serialized); } } r = 0; /* success */ done: return r; } PARSER_Construct(Float) { int r = 0; struct data_Float *data = (struct data_Float*) calloc(1, sizeof(struct data_Float)); data->fmt_mode = FMT_AS_STRING; if(json == NULL) goto done; struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(!strcmp(key, "format")) { const char *fmtmode = json_object_get_string(val); if(!strcmp(fmtmode, "number")) { data->fmt_mode = FMT_AS_NUMBER; } else if(!strcmp(fmtmode, "string")) { data->fmt_mode = FMT_AS_STRING; } else { ln_errprintf(ctx, 0, "invalid value for float:format %s", fmtmode); } } else { if(!(strcmp(key, "name") == 0 && strcmp(json_object_get_string(val), "-") == 0)) { ln_errprintf(ctx, 0, "invalid param for float: %s", key); } } json_object_iter_next(&it); } done: *pdata = data; return r; } PARSER_Destruct(Float) { free(pdata); } struct data_HexNumber { uint64_t maxval; enum FMT_MODE fmt_mode; }; /** * Parse a hex Number. * A hex number begins with 0x and contains only hex digits until the terminating * whitespace. Note that if a non-hex character is detected inside the number string, * this is NOT considered to be a number. */ PARSER_Parse(HexNumber) const char *c; size_t i = *offs; struct data_HexNumber *const data = (struct data_HexNumber*) pdata; uint64_t maxval = data->maxval; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; if(c[i] != '0' || c[i+1] != 'x') goto done; uint64_t val = 0; for (i += 2 ; i < npb->strLen && isxdigit(c[i]); i++) { const char digit = tolower(c[i]); val *= 16; if(digit >= 'a' && digit <= 'f') val += digit - 'a' + 10; else val += digit - '0'; } if (i == *offs || !isspace(c[i])) goto done; if(maxval > 0 && val > maxval) { LN_DBGPRINTF(npb->ctx, "hexnumber parser: val too large (max %" PRIu64 ", actual %" PRIu64 ")", maxval, val); goto done; } /* success, persist */ *parsed = i - *offs; if(value != NULL) { if(data->fmt_mode == FMT_AS_STRING) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } else { *value = json_object_new_int64((int64_t) val); } } r = 0; /* success */ done: return r; } PARSER_Construct(HexNumber) { int r = 0; struct data_HexNumber *data = (struct data_HexNumber*) calloc(1, sizeof(struct data_HexNumber)); data->fmt_mode = FMT_AS_STRING; if(json == NULL) goto done; struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(!strcmp(key, "maxval")) { errno = 0; data->maxval = json_object_get_int64(val); if(errno != 0) { ln_errprintf(ctx, errno, "param 'maxval' must be integer but is: %s", json_object_to_json_string(val)); } } else if(!strcmp(key, "format")) { const char *fmtmode = json_object_get_string(val); if(!strcmp(fmtmode, "number")) { data->fmt_mode = FMT_AS_NUMBER; } else if(!strcmp(fmtmode, "string")) { data->fmt_mode = FMT_AS_STRING; } else { ln_errprintf(ctx, 0, "invalid value for hexnumber:format %s", fmtmode); } } else { if(!(strcmp(key, "name") == 0 && strcmp(json_object_get_string(val), "-") == 0)) { ln_errprintf(ctx, 0, "invalid param for hexnumber: %s", key); } } json_object_iter_next(&it); } done: *pdata = data; return r; } PARSER_Destruct(HexNumber) { free(pdata); } /** * Parse a kernel timestamp. * This is a fixed format, see * https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/kernel/printk/printk.c?id=refs/tags/v4.0#n1011 * This is the code that generates it: * sprintf(buf, "[%5lu.%06lu] ", (unsigned long)ts, rem_nsec / 1000); * We accept up to 12 digits for ts, everything above that for sure is * no timestamp. */ #define LEN_KERNEL_TIMESTAMP 14 PARSER_Parse(KernelTimestamp) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; if(c[i] != '[' || i+LEN_KERNEL_TIMESTAMP > npb->strLen || !myisdigit(c[i+1]) || !myisdigit(c[i+2]) || !myisdigit(c[i+3]) || !myisdigit(c[i+4]) || !myisdigit(c[i+5]) ) goto done; i += 6; for(int j = 0 ; j < 7 && i < npb->strLen && myisdigit(c[i]) ; ) ++i, ++j; /* just scan */ if(i >= npb->strLen || c[i] != '.') goto done; ++i; /* skip over '.' */ if( i+7 > npb->strLen || !myisdigit(c[i+0]) || !myisdigit(c[i+1]) || !myisdigit(c[i+2]) || !myisdigit(c[i+3]) || !myisdigit(c[i+4]) || !myisdigit(c[i+5]) || c[i+6] != ']' ) goto done; i += 7; /* success, persist */ *parsed = i - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } /** * Parse whitespace. * This parses all whitespace until the first non-whitespace character * is found. This is primarily a tool to skip to the next "word" if * the exact number of whitespace characters (and type of whitespace) * is not known. The current parsing position MUST be on a whitespace, * else the parser does not match. * This parser is also a forward-compatibility tool for the upcoming * slsa (simple log structure analyser) tool. */ PARSER_Parse(Whitespace) const char *c; size_t i = *offs; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; if(!isspace(c[i])) goto done; for (i++ ; i < npb->strLen && isspace(c[i]); i++); /* success, persist */ *parsed = i - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } /** * Parse a word. * A word is a SP-delimited entity. The parser always works, except if * the offset is position on a space upon entry. */ PARSER_Parse(Word) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; /* search end of word */ while(i < npb->strLen && c[i] != ' ') i++; if(i == *offs) goto done; /* success, persist */ *parsed = i - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } struct data_StringTo { const char *toFind; size_t len; }; /** * Parse everything up to a specific string. * swisskid, 2015-01-21 */ PARSER_Parse(StringTo) const char *c; size_t i, j, m; int chkstr; struct data_StringTo *const data = (struct data_StringTo*) pdata; const char *const toFind = data->toFind; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; chkstr = 0; /* Total hunt for letter */ while(chkstr == 0 && i < npb->strLen ) { i++; if(c[i] == toFind[0]) { /* Found the first letter, now find the rest of the string */ j = 1; m = i+1; while(m < npb->strLen && j < data->len ) { if(c[m] != toFind[j]) break; if(j == data->len - 1) { /* full match? */ chkstr = 1; break; } j++; m++; } } } if(i == *offs || i == npb->strLen || chkstr != 1) goto done; /* success, persist */ *parsed = i - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } PARSER_Construct(StringTo) { int r = 0; struct data_StringTo *data = (struct data_StringTo*) calloc(1, sizeof(struct data_StringTo)); struct json_object *ed; if(json_object_object_get_ex(json, "extradata", &ed) == 0) { ln_errprintf(ctx, 0, "string-to type needs 'extradata' parameter"); r = LN_BADCONFIG ; goto done; } data->toFind = strdup(json_object_get_string(ed)); data->len = strlen(data->toFind); *pdata = data; done: if(r != 0) free(data); return r; } PARSER_Destruct(StringTo) { struct data_StringTo *data = (struct data_StringTo*) pdata; free((void*)data->toFind); free(pdata); } /** * Parse a alphabetic word. * A alpha word is composed of characters for which isalpha returns true. * The parser dones if there is no alpha character at all. */ PARSER_Parse(Alpha) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; /* search end of word */ while(i < npb->strLen && isalpha(c[i])) i++; if(i == *offs) { goto done; } /* success, persist */ *parsed = i - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } struct data_CharTo { char *term_chars; size_t n_term_chars; char *data_for_display; }; /** * Parse everything up to a specific character. * The character must be the only char inside extra data passed to the parser. * It is considered a format error if * a) the to-be-parsed buffer is already positioned on the terminator character * b) there is no terminator until the end of the buffer * In those cases, the parsers declares itself as not being successful, in all * other cases a string is extracted. */ PARSER_Parse(CharTo) size_t i; struct data_CharTo *const data = (struct data_CharTo*) pdata; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); i = *offs; /* search end of word */ int found = 0; while(i < npb->strLen && !found) { for(size_t j = 0 ; j < data->n_term_chars ; ++j) { if(npb->str[i] == data->term_chars[j]) { found = 1; break; } } if(!found) ++i; } if(i == *offs || i == npb->strLen || !found) goto done; /* success, persist */ *parsed = i - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; done: return r; } PARSER_Construct(CharTo) { int r = 0; LN_DBGPRINTF(ctx, "in parser_construct charTo"); struct data_CharTo *data = (struct data_CharTo*) calloc(1, sizeof(struct data_CharTo)); struct json_object *ed; if(json_object_object_get_ex(json, "extradata", &ed) == 0) { ln_errprintf(ctx, 0, "char-to type needs 'extradata' parameter"); r = LN_BADCONFIG ; goto done; } data->term_chars = strdup(json_object_get_string(ed)); data->n_term_chars = strlen(data->term_chars); *pdata = data; done: if(r != 0) free(data); return r; } PARSER_DataForDisplay(CharTo) { struct data_CharTo *data = (struct data_CharTo*) pdata; if(data->data_for_display == NULL) { data->data_for_display = malloc(8+data->n_term_chars+2); if(data->data_for_display != NULL) { memcpy(data->data_for_display, "char-to{", 8); size_t i, j; for(j = 0, i = 8 ; j < data->n_term_chars ; ++j, ++i) { data->data_for_display[i] = data->term_chars[j]; } data->data_for_display[i++] = '}'; data->data_for_display[i] = '\0'; } } return (data->data_for_display == NULL ) ? "malloc error" : data->data_for_display; } PARSER_Destruct(CharTo) { struct data_CharTo *const data = (struct data_CharTo*) pdata; free(data->data_for_display); free(data->term_chars); free(pdata); } struct data_Literal { const char *lit; const char *json_conf; }; /** * Parse a specific literal. */ PARSER_Parse(Literal) struct data_Literal *const data = (struct data_Literal*) pdata; const char *const lit = data->lit; size_t i = *offs; size_t j; for(j = 0 ; i < npb->strLen ; ++j) { if(lit[j] != npb->str[i]) break; ++i; } *parsed = j; /* we must always return how far we parsed! */ if(lit[j] == '\0') { if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; } return r; } PARSER_DataForDisplay(Literal) { struct data_Literal *data = (struct data_Literal*) pdata; return data->lit; } PARSER_JsonConf(Literal) { struct data_Literal *data = (struct data_Literal*) pdata; return data->json_conf; } PARSER_Construct(Literal) { int r = 0; struct data_Literal *data = (struct data_Literal*) calloc(1, sizeof(struct data_Literal)); struct json_object *text; if(json_object_object_get_ex(json, "text", &text) == 0) { ln_errprintf(ctx, 0, "literal type needs 'text' parameter"); r = LN_BADCONFIG ; goto done; } data->lit = strdup(json_object_get_string(text)); data->json_conf = strdup(json_object_to_json_string(json)); *pdata = data; done: if(r != 0) free(data); return r; } PARSER_Destruct(Literal) { struct data_Literal *data = (struct data_Literal*) pdata; free((void*)data->lit); free((void*)data->json_conf); free(pdata); } /* for path compaction, we need a special handler to combine two * literal data elements. */ int ln_combineData_Literal(void *const porg, void *const padd) { struct data_Literal *const __restrict__ org = porg; struct data_Literal *const __restrict__ add = padd; int r = 0; const size_t len = strlen(org->lit); const size_t add_len = strlen(add->lit); char *const newlit = (char*)realloc((void*)org->lit, len+add_len+1); CHKN(newlit); org->lit = newlit; memcpy((char*)org->lit+len, add->lit, add_len+1); done: return r; } struct data_CharSeparated { char *term_chars; size_t n_term_chars; }; /** * Parse everything up to a specific character, or up to the end of string. * The character must be the only char inside extra data passed to the parser. * This parser always returns success. * By nature of the parser, it is required that end of string or the separator * follows this field in rule. */ PARSER_Parse(CharSeparated) struct data_CharSeparated *const data = (struct data_CharSeparated*) pdata; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); i = *offs; /* search end of word */ int found = 0; while(i < npb->strLen && !found) { for(size_t j = 0 ; j < data->n_term_chars ; ++j) { if(npb->str[i] == data->term_chars[j]) { found = 1; break; } } if(!found) ++i; } /* success, persist */ *parsed = i - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ return r; } PARSER_Construct(CharSeparated) { int r = 0; struct data_CharSeparated *data = (struct data_CharSeparated*) calloc(1, sizeof(struct data_CharSeparated)); struct json_object *ed; if(json_object_object_get_ex(json, "extradata", &ed) == 0) { ln_errprintf(ctx, 0, "char-separated type needs 'extradata' parameter"); r = LN_BADCONFIG ; goto done; } data->term_chars = strdup(json_object_get_string(ed)); data->n_term_chars = strlen(data->term_chars); *pdata = data; done: if(r != 0) free(data); return r; } PARSER_Destruct(CharSeparated) { struct data_CharSeparated *const data = (struct data_CharSeparated*) pdata; free(data->term_chars); free(pdata); } /** * Just get everything till the end of string. */ PARSER_Parse(Rest) assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); /* silence the warning about unused variable */ (void)npb->str; /* success, persist */ *parsed = npb->strLen - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; return r; } /** * Parse a possibly quoted string. In this initial implementation, escaping of the quote * char is not supported. A quoted string is one start starts with a double quote, * has some text (not containing double quotes) and ends with the first double * quote character seen. The extracted string does NOT include the quote characters. * swisskid, 2015-01-21 */ PARSER_Parse(OpQuotedString) const char *c; size_t i; char *cstr = NULL; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; if(c[i] != '"') { while(i < npb->strLen && c[i] != ' ') i++; if(i == *offs) goto done; /* success, persist */ *parsed = i - *offs; /* create JSON value to save quoted string contents */ CHKN(cstr = strndup((char*)c + *offs, *parsed)); } else { ++i; /* search end of string */ while(i < npb->strLen && c[i] != '"') i++; if(i == npb->strLen || c[i] != '"') goto done; /* success, persist */ *parsed = i + 1 - *offs; /* "eat" terminal double quote */ /* create JSON value to save quoted string contents */ CHKN(cstr = strndup((char*)c + *offs + 1, *parsed - 2)); } CHKN(*value = json_object_new_string(cstr)); r = 0; /* success */ done: free(cstr); return r; } /** * Parse a quoted string. In this initial implementation, escaping of the quote * char is not supported. A quoted string is one start starts with a double quote, * has some text (not containing double quotes) and ends with the first double * quote character seen. The extracted string does NOT include the quote characters. * rgerhards, 2011-01-14 */ PARSER_Parse(QuotedString) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; if(i + 2 > npb->strLen) goto done; /* needs at least 2 characters */ if(c[i] != '"') goto done; ++i; /* search end of string */ while(i < npb->strLen && c[i] != '"') i++; if(i == npb->strLen || c[i] != '"') goto done; /* success, persist */ *parsed = i + 1 - *offs; /* "eat" terminal double quote */ /* create JSON value to save quoted string contents */ if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } /** * Parse an ISO date, that is YYYY-MM-DD (exactly this format). * Note: we do manual loop unrolling -- this is fast AND efficient. * rgerhards, 2011-01-14 */ PARSER_Parse(ISODate) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; if(*offs+10 > npb->strLen) goto done; /* if it is not 10 chars, it can't be an ISO date */ /* year */ if(!myisdigit(c[i])) goto done; if(!myisdigit(c[i+1])) goto done; if(!myisdigit(c[i+2])) goto done; if(!myisdigit(c[i+3])) goto done; if(c[i+4] != '-') goto done; /* month */ if(c[i+5] == '0') { if(c[i+6] < '1' || c[i+6] > '9') goto done; } else if(c[i+5] == '1') { if(c[i+6] < '0' || c[i+6] > '2') goto done; } else { goto done; } if(c[i+7] != '-') goto done; /* day */ if(c[i+8] == '0') { if(c[i+9] < '1' || c[i+9] > '9') goto done; } else if(c[i+8] == '1' || c[i+8] == '2') { if(!myisdigit(c[i+9])) goto done; } else if(c[i+8] == '3') { if(c[i+9] != '0' && c[i+9] != '1') goto done; } else { goto done; } /* success, persist */ *parsed = 10; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } /** * Parse a Cisco interface spec. Sample for such a spec are: * outside:192.168.52.102/50349 * inside:192.168.1.15/56543 (192.168.1.112/54543) * outside:192.168.1.13/50179 (192.168.1.13/50179)(LOCAL\some.user) * outside:192.168.1.25/41850(LOCAL\RG-867G8-DEL88D879BBFFC8) * inside:192.168.1.25/53 (192.168.1.25/53) (some.user) * 192.168.1.15/0(LOCAL\RG-867G8-DEL88D879BBFFC8) * From this, we conclude the format is: * [interface:]ip/port [SP (ip2/port2)] [[SP](username)] * In order to match, this syntax must start on a non-whitespace char * other than colon. */ PARSER_Parse(CiscoInterfaceSpec) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; if(c[i] == ':' || isspace(c[i])) goto done; /* first, check if we have an interface. We do this by trying * to detect if we have an IP. If we have, obviously no interface * is present. Otherwise, we check if we have a valid interface. */ int bHaveInterface = 0; size_t idxInterface = 0; size_t lenInterface = 0; int bHaveIP = 0; size_t lenIP; size_t idxIP = i; if(ln_v2_parseIPv4(npb, &i, NULL, &lenIP, NULL) == 0) { bHaveIP = 1; i += lenIP - 1; /* position on delimiter */ } else { idxInterface = i; while(i < npb->strLen) { if(isspace(c[i])) goto done; if(c[i] == ':') break; ++i; } lenInterface = i - idxInterface; bHaveInterface = 1; } if(i == npb->strLen) goto done; ++i; /* skip over colon */ /* we now utilize our other parser helpers */ if(!bHaveIP) { idxIP = i; if(ln_v2_parseIPv4(npb, &i, NULL, &lenIP, NULL) != 0) goto done; i += lenIP; } if(i == npb->strLen || c[i] != '/') goto done; ++i; /* skip slash */ const size_t idxPort = i; size_t lenPort; if(ln_v2_parseNumber(npb, &i, NULL, &lenPort, NULL) != 0) goto done; i += lenPort; /* check if optional second ip/port is present * We assume we must at least have 5 chars [" (::1)"] */ int bHaveIP2 = 0; size_t idxIP2 = 0, lenIP2 = 0; size_t idxPort2 = 0, lenPort2 = 0; if(i+5 < npb->strLen && c[i] == ' ' && c[i+1] == '(') { size_t iTmp = i+2; /* skip over " (" */ idxIP2 = iTmp; if(ln_v2_parseIPv4(npb, &iTmp, NULL, &lenIP2, NULL) == 0) { iTmp += lenIP2; if(i < npb->strLen || c[iTmp] == '/') { ++iTmp; /* skip slash */ idxPort2 = iTmp; if(ln_v2_parseNumber(npb, &iTmp, NULL, &lenPort2, NULL) == 0) { iTmp += lenPort2; if(iTmp < npb->strLen && c[iTmp] == ')') { i = iTmp + 1; /* match, so use new index */ bHaveIP2 = 1; } } } } } /* check if optional username is present * We assume we must at least have 3 chars ["(n)"] */ int bHaveUser = 0; size_t idxUser = 0; size_t lenUser = 0; if( (i+2 < npb->strLen && c[i] == '(' && !isspace(c[i+1]) ) || (i+3 < npb->strLen && c[i] == ' ' && c[i+1] == '(' && !isspace(c[i+2])) ) { idxUser = i + ((c[i] == ' ') ? 2 : 1); /* skip [SP]'(' */ size_t iTmp = idxUser; while(iTmp < npb->strLen && !isspace(c[iTmp]) && c[iTmp] != ')') ++iTmp; /* just scan */ if(iTmp < npb->strLen && c[iTmp] == ')') { i = iTmp + 1; /* we have a match, so use new index */ bHaveUser = 1; lenUser = iTmp - idxUser; } } /* all done, save data */ if(value == NULL) goto success; CHKN(*value = json_object_new_object()); json_object *json; if(bHaveInterface) { CHKN(json = json_object_new_string_len(c+idxInterface, lenInterface)); json_object_object_add_ex(*value, "interface", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); } CHKN(json = json_object_new_string_len(c+idxIP, lenIP)); json_object_object_add_ex(*value, "ip", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); CHKN(json = json_object_new_string_len(c+idxPort, lenPort)); json_object_object_add_ex(*value, "port", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); if(bHaveIP2) { CHKN(json = json_object_new_string_len(c+idxIP2, lenIP2)); json_object_object_add_ex(*value, "ip2", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); CHKN(json = json_object_new_string_len(c+idxPort2, lenPort2)); json_object_object_add_ex(*value, "port2", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); } if(bHaveUser) { CHKN(json = json_object_new_string_len(c+idxUser, lenUser)); json_object_object_add_ex(*value, "user", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); } success: /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: if(r != 0 && value != NULL && *value != NULL) { json_object_put(*value); *value = NULL; /* to be on the save side */ } return r; } /** * Parse a duration. A duration is similar to a timestamp, except that * it tells about time elapsed. As such, hours can be larger than 23 * and hours may also be specified by a single digit (this, for example, * is commonly done in Cisco software). * Note: we do manual loop unrolling -- this is fast AND efficient. */ PARSER_Parse(Duration) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; /* hour is a bit tricky */ if(!myisdigit(c[i])) goto done; ++i; if(myisdigit(c[i])) ++i; if(c[i] == ':') ++i; else goto done; if(i+5 > npb->strLen) goto done;/* if it is not 5 chars from here, it can't be us */ if(c[i] < '0' || c[i] > '5') goto done; if(!myisdigit(c[i+1])) goto done; if(c[i+2] != ':') goto done; if(c[i+3] < '0' || c[i+3] > '5') goto done; if(!myisdigit(c[i+4])) goto done; /* success, persist */ *parsed = (i + 5) - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } /** * Parse a timestamp in 24hr format (exactly HH:MM:SS). * Note: we do manual loop unrolling -- this is fast AND efficient. * rgerhards, 2011-01-14 */ PARSER_Parse(Time24hr) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; if(*offs+8 > npb->strLen) goto done; /* if it is not 8 chars, it can't be us */ /* hour */ if(c[i] == '0' || c[i] == '1') { if(!myisdigit(c[i+1])) goto done; } else if(c[i] == '2') { if(c[i+1] < '0' || c[i+1] > '3') goto done; } else { goto done; } /* TODO: the code below is a duplicate of 24hr parser - create common function */ if(c[i+2] != ':') goto done; if(c[i+3] < '0' || c[i+3] > '5') goto done; if(!myisdigit(c[i+4])) goto done; if(c[i+5] != ':') goto done; if(c[i+6] < '0' || c[i+6] > '5') goto done; if(!myisdigit(c[i+7])) goto done; /* success, persist */ *parsed = 8; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } /** * Parse a timestamp in 12hr format (exactly HH:MM:SS). * Note: we do manual loop unrolling -- this is fast AND efficient. * TODO: the code below is a duplicate of 24hr parser - create common function? * rgerhards, 2011-01-14 */ PARSER_Parse(Time12hr) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); c = npb->str; i = *offs; if(*offs+8 > npb->strLen) goto done; /* if it is not 8 chars, it can't be us */ /* hour */ if(c[i] == '0') { if(!myisdigit(c[i+1])) goto done; } else if(c[i] == '1') { if(c[i+1] < '0' || c[i+1] > '2') goto done; } else { goto done; } if(c[i+2] != ':') goto done; if(c[i+3] < '0' || c[i+3] > '5') goto done; if(!myisdigit(c[i+4])) goto done; if(c[i+5] != ':') goto done; if(c[i+6] < '0' || c[i+6] > '5') goto done; if(!myisdigit(c[i+7])) goto done; /* success, persist */ *parsed = 8; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } /* helper to IPv4 address parser, checks the next set of numbers. * Syntax 1 to 3 digits, value together not larger than 255. * @param[in] npb->str parse buffer * @param[in/out] offs offset into buffer, updated if successful * @return 0 if OK, 1 otherwise */ static int chkIPv4AddrByte(npb_t *const npb, size_t *offs) { int val = 0; int r = 1; /* default: done -- simplifies things */ const char *c; size_t i = *offs; c = npb->str; if(i == npb->strLen || !myisdigit(c[i])) goto done; val = c[i++] - '0'; if(i < npb->strLen && myisdigit(c[i])) { val = val * 10 + c[i++] - '0'; if(i < npb->strLen && myisdigit(c[i])) val = val * 10 + c[i++] - '0'; } if(val > 255) /* cannot be a valid IP address byte! */ goto done; *offs = i; r = 0; done: return r; } /** * Parser for IPv4 addresses. */ PARSER_Parse(IPv4) const char *c; size_t i; assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); i = *offs; if(i + 7 > npb->strLen) { /* IPv4 addr requires at least 7 characters */ goto done; } c = npb->str; /* byte 1*/ if(chkIPv4AddrByte(npb, &i) != 0) goto done; if(i == npb->strLen || c[i++] != '.') goto done; /* byte 2*/ if(chkIPv4AddrByte(npb, &i) != 0) goto done; if(i == npb->strLen || c[i++] != '.') goto done; /* byte 3*/ if(chkIPv4AddrByte(npb, &i) != 0) goto done; if(i == npb->strLen || c[i++] != '.') goto done; /* byte 4 - we do NOT need any char behind it! */ if(chkIPv4AddrByte(npb, &i) != 0) goto done; /* if we reach this point, we found a valid IP address */ *parsed = i - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } /* skip past the IPv6 address block, parse pointer is set to * first char after the block. Returns an error if already at end * of string. * @param[in] npb->str parse buffer * @param[in/out] offs offset into buffer, updated if successful * @return 0 if OK, 1 otherwise */ static int skipIPv6AddrBlock(npb_t *const npb, size_t *const __restrict__ offs) { int j; if(*offs == npb->strLen) return 1; for(j = 0 ; j < 4 && *offs+j < npb->strLen && isxdigit(npb->str[*offs+j]) ; ++j) /*just skip*/ ; *offs += j; return 0; } /** * Parser for IPv6 addresses. * Bases on RFC4291 Section 2.2. The address must be followed * by whitespace or end-of-string, else it is not considered * a valid address. This prevents false positives. */ PARSER_Parse(IPv6) const char *c; size_t i; size_t beginBlock; /* last block begin in case we need IPv4 parsing */ int hasIPv4 = 0; int nBlocks = 0; /* how many blocks did we already have? */ int bHad0Abbrev = 0; /* :: already used? */ assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); i = *offs; if(i + 2 > npb->strLen) { /* IPv6 addr requires at least 2 characters ("::") */ goto done; } c = npb->str; /* check that first block is non-empty */ if(! ( isxdigit(c[i]) || (c[i] == ':' && c[i+1] == ':') ) ) goto done; /* try for all potential blocks plus one more (so we see errors!) */ for(int j = 0 ; j < 9 ; ++j) { beginBlock = i; if(skipIPv6AddrBlock(npb, &i) != 0) goto done; nBlocks++; if(i == npb->strLen) goto chk_ok; if(isspace(c[i])) goto chk_ok; if(c[i] == '.'){ /* IPv4 processing! */ hasIPv4 = 1; break; } if(c[i] != ':') goto done; i++; /* "eat" ':' */ if(i == npb->strLen) goto chk_ok; /* check for :: */ if(bHad0Abbrev) { if(c[i] == ':') goto done; } else { if(c[i] == ':') { bHad0Abbrev = 1; ++i; if(i == npb->strLen) goto chk_ok; } } } if(hasIPv4) { size_t ipv4_parsed; --nBlocks; /* prevent pure IPv4 address to be recognized */ if(beginBlock == *offs) goto done; i = beginBlock; if(ln_v2_parseIPv4(npb, &i, NULL, &ipv4_parsed, NULL) != 0) goto done; i += ipv4_parsed; } chk_ok: /* we are finished parsing, check if things are ok */ if(nBlocks > 8) goto done; if(bHad0Abbrev && nBlocks >= 8) goto done; /* now check if trailing block is missing. Note that i is already * on next character, so we need to go two back. Two are always * present, else we would not reach this code here. */ if(c[i-1] == ':' && c[i-2] != ':') goto done; /* if we reach this point, we found a valid IP address */ *parsed = i - *offs; if(value != NULL) { *value = json_object_new_string_len(npb->str+(*offs), *parsed); } r = 0; /* success */ done: return r; } /* check if a char is valid inside a name of the iptables motif. * We try to keep the set as slim as possible, because the iptables * parser may otherwise create a very broad match (especially the * inclusion of simple words like "DF" cause grief here). * Note: we have taken the permitted set from iptables log samples. * Report bugs if we missed some additional rules. */ static inline int isValidIPTablesNameChar(const char c) { /* right now, upper case only is valid */ return ('A' <= c && c <= 'Z') ? 1 : 0; } /* helper to iptables parser, parses out a a single name=value pair */ static int parseIPTablesNameValue(npb_t *const npb, size_t *const __restrict__ offs, struct json_object *const __restrict__ valroot) { int r = LN_WRONGPARSER; size_t i = *offs; char *name = NULL; const size_t iName = i; while(i < npb->strLen && isValidIPTablesNameChar(npb->str[i])) ++i; if(i == iName || (i < npb->strLen && npb->str[i] != '=' && npb->str[i] != ' ')) goto done; /* no name at all! */ const ssize_t lenName = i - iName; ssize_t iVal = -1; size_t lenVal = i - iVal; if(i < npb->strLen && npb->str[i] != ' ') { /* we have a real value (not just a flag name like "DF") */ ++i; /* skip '=' */ iVal = i; while(i < npb->strLen && !isspace(npb->str[i])) ++i; lenVal = i - iVal; } /* parsing OK */ *offs = i; r = 0; if(valroot == NULL) goto done; CHKN(name = malloc(lenName+1)); memcpy(name, npb->str+iName, lenName); name[lenName] = '\0'; json_object *json; if(iVal == -1) { json = NULL; } else { CHKN(json = json_object_new_string_len(npb->str+iVal, lenVal)); } json_object_object_add(valroot, name, json); done: free(name); return r; } /** * Parser for iptables logs (the structured part). * This parser is named "v2-iptables" because of a traditional * parser named "iptables", which we do not want to replace, at * least right now (we may re-think this before the first release). * For performance reasons, this works in two stages. In the first * stage, we only detect if the motif is correct. The second stage is * only called when we know it is. In it, we go once again over the * message again and actually extract the data. This is done because * data extraction is relatively expensive and in most cases we will * have much more frequent mismatches than matches. * Note that this motif must have at least one field, otherwise it * could detect things that are not iptables to be it. Further limits * may be imposed in the future as we see additional need. * added 2015-04-30 rgerhards */ PARSER_Parse(v2IPTables) size_t i = *offs; int nfields = 0; /* stage one */ while(i < npb->strLen) { CHKR(parseIPTablesNameValue(npb, &i, NULL)); ++nfields; /* exactly one SP is permitted between fields */ if(i < npb->strLen && npb->str[i] == ' ') ++i; } if(nfields < 2) { FAIL(LN_WRONGPARSER); } /* success, persist */ *parsed = i - *offs; r = 0; /* stage two */ if(value == NULL) goto done; i = *offs; CHKN(*value = json_object_new_object()); while(i < npb->strLen) { CHKR(parseIPTablesNameValue(npb, &i, *value)); while(i < npb->strLen && isspace(npb->str[i])) ++i; } done: if(r != 0 && value != NULL && *value != NULL) { json_object_put(*value); *value = NULL; } return r; } /** * Parse JSON. This parser tries to find JSON data inside a message. * If it finds valid JSON, it will extract it. Extra data after the * JSON is permitted. * Note: the json-c JSON parser treats whitespace after the actual * json to be part of the json. So in essence, any whitespace is * processed by this parser. We use the same semantics to keep things * neatly in sync. If json-c changes for some reason or we switch to * an alternate json lib, we probably need to be sure to keep that * behaviour, and probably emulate it. * added 2015-04-28 by rgerhards, v1.1.2 */ PARSER_Parse(JSON) const size_t i = *offs; struct json_tokener *tokener = NULL; if(npb->str[i] != '{' && npb->str[i] != ']') { /* this can't be json, see RFC4627, Sect. 2 * see this bug in json-c: * https://github.com/json-c/json-c/issues/181 * In any case, it's better to do this quick check, * even if json-c did not have the bug because this * check here is much faster than calling the parser. */ goto done; } if((tokener = json_tokener_new()) == NULL) goto done; struct json_object *const json = json_tokener_parse_ex(tokener, npb->str+i, (int) (npb->strLen - i)); if(json == NULL) goto done; /* success, persist */ *parsed = (i + tokener->char_offset) - *offs; r = 0; /* success */ if(value == NULL) { json_object_put(json); } else { *value = json; } done: if(tokener != NULL) json_tokener_free(tokener); return r; } /* check if a char is valid inside a name of a NameValue list * The set of valid characters may be extended if there is good * need to do so. We have selected the current set carefully, but * may have overlooked some cases. */ static inline int isValidNameChar(const char c) { return (isalnum(c) || c == '.' || c == '_' || c == '-' ) ? 1 : 0; } /* helper to NameValue parser, parses out a a single name=value pair * * name must be alphanumeric characters, value must be non-whitespace * characters, if quoted than with symmetric quotes. Supported formats * - name=value * - name="value" * - name='value' * Note "name=" is valid and means a field with empty value. * TODO: so far, quote characters are not permitted WITHIN quoted values. */ static int parseNameValue(npb_t *const npb, size_t *const __restrict__ offs, struct json_object *const __restrict__ valroot, const char sep, const char ass) { int r = LN_WRONGPARSER; size_t i = *offs; char *name = NULL; const size_t iName = i; /* If the assignator character is specified, search for it If it's not, check key name validity */ while(i < npb->strLen && ((ass != 0) ? (npb->str[i] != ass) : isValidNameChar(npb->str[i]))) ++i; if(i == iName || ((ass != 0) ? (npb->str[i] != ass) : (npb->str[i] != '='))) goto done; /* no name at all! */ const size_t lenName = i - iName; ++i; /* skip assignator */ char quoting = npb->str[i]; if(i < npb->strLen && (quoting == '"' || quoting == '\'')) i++; else quoting = 0; //no quoting detected const size_t iVal = i; if(quoting) { // wait on an unescaped matching quoting /* * Fix by HSoszynski & KGuillemot to handle escaped quote & backslash infinitly * Continue while we don't encounter the ending quote, and while it's not escaped * a" => end * a\" => continue * a\\" => end * a\\\" => continue * ... */ int continuous_backslash = 0; while(i < npb->strLen && (npb->str[i] != quoting || continuous_backslash%2 == 1 )) { if ( npb->str[i] == '\\' ) { continuous_backslash++; } else { continuous_backslash = 0; } ++i; } } else { /* We seek characters as long as: - we have characters remaining - the character is NOT a whitespace (default separator) - the character is NOT the separator set explicitely by the user (sep) - the character IS the separator (sep), BUT is escaped */ /* * Fix by HSoszynski & KGuillemot to handle escaped separator & backslash infinitly * Continue while we don't encounter the ending separator, and while it's not escaped * a, => end * a\, => continue * a\\, => end * a\\\, => continue * ... */ int continuous_backslash = 0; while(i < npb->strLen && ((sep == 0 ? (!isspace(npb->str[i])) : (npb->str[i] != sep)) || continuous_backslash%2 == 1)) { if ( npb->str[i] == '\\' ) { continuous_backslash++; } else { continuous_backslash = 0; } ++i; } } // in case of quoting, ensure we skip it if(i < npb->strLen && npb->str[i] == quoting) ++i; else if(quoting) goto done; const size_t lenVal = i - iVal - (quoting ? 1 : 0); /* parsing OK */ *offs = i; r = 0; if(valroot == NULL) goto done; CHKN(name = malloc(lenName+1)); memcpy(name, npb->str+iName, lenName); name[lenName] = '\0'; json_object *json; CHKN(json = json_object_new_string_len(npb->str+iVal, lenVal)); json_object_object_add(valroot, name, json); done: free(name); return r; } /** * Parse CEE syslog. * This essentially is a JSON parser, with additional restrictions: * The message must start with "@cee:" and json must immediately follow (whitespace permitted). * after the JSON, there must be no other non-whitespace characters. * In other words: the message must consist of a single JSON object, * only. * added 2015-04-28 by rgerhards, v1.1.2 */ PARSER_Parse(CEESyslog) size_t i = *offs; struct json_tokener *tokener = NULL; struct json_object *json = NULL; if(npb->strLen < i + 7 || /* "@cee:{}" is minimum text */ npb->str[i] != '@' || npb->str[i+1] != 'c' || npb->str[i+2] != 'e' || npb->str[i+3] != 'e' || npb->str[i+4] != ':') goto done; /* skip whitespace */ for(i += 5 ; i < npb->strLen && isspace(npb->str[i]) ; ++i) /* just skip */; if(i == npb->strLen || npb->str[i] != '{') goto done; /* note: we do not permit arrays in CEE mode */ if((tokener = json_tokener_new()) == NULL) goto done; json = json_tokener_parse_ex(tokener, npb->str+i, (int) (npb->strLen - i)); if(json == NULL) goto done; if(i + tokener->char_offset != npb->strLen) goto done; /* success, persist */ *parsed = npb->strLen; r = 0; /* success */ if(value != NULL) { *value = json; json = NULL; /* do NOT free below! */ } done: if(tokener != NULL) json_tokener_free(tokener); if(json != NULL) json_object_put(json); return r; } struct data_NameValue { char sep; /* separator (between key/value couples) */ char ass; /* assignator (between key and value) */ }; /** * Parser for name/value pairs. * On entry must point to alnum char. All following chars must be * name/value pairs delimited by whitespace up until the end of string. * For performance reasons, this works in two stages. In the first * stage, we only detect if the motif is correct. The second stage is * only called when we know it is. In it, we go once again over the * message again and actually extract the data. This is done because * data extraction is relatively expensive and in most cases we will * have much more frequent mismatches than matches. * added 2015-04-25 rgerhards */ PARSER_Parse(NameValue) size_t i = *offs; struct data_NameValue *const data = (struct data_NameValue*) pdata; const char sep = data->sep; const char ass = data->ass; LN_DBGPRINTF(npb->ctx, "in parse_NameValue, separator is '%c'(0x%02x) assignator is '%c'(0x%02x)" ,sep, sep, ass, ass); /* stage one */ while(i < npb->strLen) { if (parseNameValue(npb, &i, NULL, sep, ass) == 0 ) { // Check if there is at least one time the separator after value if( i < npb->strLen && !(sep == 0 ? (isspace(npb->str[i])) : (npb->str[i] == sep)) ) break; while(i < npb->strLen && (sep == 0 ? (isspace(npb->str[i])) : (npb->str[i] == sep))) ++i; } else { break; } } /* success, persist */ *parsed = i - *offs; r = 0; /* success */ /* stage two */ if(value == NULL) goto done; i = *offs; CHKN(*value = json_object_new_object()); while(i < npb->strLen) { if (parseNameValue(npb, &i, *value, sep, ass) == 0 ) { // Check if there is at least one time the separator after value if( i < npb->strLen && !(sep == 0 ? (isspace(npb->str[i])) : (npb->str[i] == sep)) ) break; while(i < npb->strLen && ((sep == 0) ? (isspace(npb->str[i])) : (npb->str[i] == sep))) ++i; } else { break; } } /* TODO: fix mem leak if alloc json fails */ done: return r; } PARSER_Construct(NameValue) { int r = 0; LN_DBGPRINTF(ctx, "in parser_construct NameValue"); struct data_NameValue *data = (struct data_NameValue*) calloc(1, sizeof(struct data_NameValue)); struct json_object *obj; const char *str; if(json_object_object_get_ex(json, "extradata", &obj) != 0) { LN_DBGPRINTF(ctx, "found 'extradata' in fields, assigning to 'separator'"); if(json_object_get_string_len(obj) == 1) { str = json_object_get_string(obj); data->sep = str[0]; } else { ln_errprintf(ctx, 0, "name-value-list's extradata should only be 1 character"); r = LN_BADCONFIG; goto done; } } if(json_object_object_get_ex(json, "separator", &obj) != 0) { LN_DBGPRINTF(ctx, "found 'separator' in fields"); if(json_object_get_string_len(obj) == 1) { str = json_object_get_string(obj); data->sep = str[0]; } else { ln_errprintf(ctx, 0, "name-value-list's 'separator' field should only be 1 character"); r = LN_BADCONFIG; goto done; } } if(json_object_object_get_ex(json, "assignator", &obj) != 0) { LN_DBGPRINTF(ctx, "found 'assignator' in fields"); if(json_object_get_string_len(obj) == 1) { str = json_object_get_string(obj); data->ass = str[0]; } else { ln_errprintf(ctx, 0, "name-value-list's 'assignator' field should only be 1 character"); r = LN_BADCONFIG; goto done; } } *pdata = data; done: if(r != 0) free(data); return r; } PARSER_Destruct(NameValue) { free(pdata); } /** * Parse a MAC layer address. * The standard (IEEE 802) format for printing MAC-48 addresses in * human-friendly form is six groups of two hexadecimal digits, * separated by hyphens (-) or colons (:), in transmission order * (e.g. 01-23-45-67-89-ab or 01:23:45:67:89:ab ). * This form is also commonly used for EUI-64. * from: http://en.wikipedia.org/wiki/MAC_address * * This parser must start on a hex digit. * added 2015-05-04 by rgerhards, v1.1.2 */ PARSER_Parse(MAC48) size_t i = *offs; char delim; if(npb->strLen < i + 17 || /* this motif has exactly 17 characters */ !isxdigit(npb->str[i]) || !isxdigit(npb->str[i+1]) ) FAIL(LN_WRONGPARSER); if(npb->str[i+2] == ':') delim = ':'; else if(npb->str[i+2] == '-') delim = '-'; else FAIL(LN_WRONGPARSER); /* first byte ok */ if(!isxdigit(npb->str[i+3]) || !isxdigit(npb->str[i+4]) || npb->str[i+5] != delim || /* 2nd byte ok */ !isxdigit(npb->str[i+6]) || !isxdigit(npb->str[i+7]) || npb->str[i+8] != delim || /* 3rd byte ok */ !isxdigit(npb->str[i+9]) || !isxdigit(npb->str[i+10]) || npb->str[i+11] != delim || /* 4th byte ok */ !isxdigit(npb->str[i+12]) || !isxdigit(npb->str[i+13]) || npb->str[i+14] != delim || /* 5th byte ok */ !isxdigit(npb->str[i+15]) || !isxdigit(npb->str[i+16]) /* 6th byte ok */ ) FAIL(LN_WRONGPARSER); /* success, persist */ *parsed = 17; r = 0; /* success */ if(value != NULL) { CHKN(*value = json_object_new_string_len(npb->str+i, 17)); } done: return r; } /* This parses the extension value and updates the index * to point to the end of it. */ static int cefParseExtensionValue(npb_t *const npb, size_t *__restrict__ iEndVal) { int r = 0; size_t i = *iEndVal; size_t iLastWordBegin; /* first find next unquoted equal sign and record begin of * last word in front of it - this is the actual end of the * current name/value pair and the begin of the next one. */ int hadSP = 0; int inEscape = 0; for(iLastWordBegin = 0 ; i < npb->strLen ; ++i) { if(inEscape) { if(npb->str[i] != '=' && npb->str[i] != '\\' && npb->str[i] != 'r' && npb->str[i] != 'n' && npb->str[i] != '/') FAIL(LN_WRONGPARSER); inEscape = 0; } else { if(npb->str[i] == '=') { break; } else if(npb->str[i] == '\\') { inEscape = 1; } else if(npb->str[i] == ' ') { hadSP = 1; } else { if(hadSP) { iLastWordBegin = i; hadSP = 0; } } } } /* Note: iLastWordBegin can never be at offset zero, because * the CEF header starts there! */ if(i < npb->strLen) { *iEndVal = (iLastWordBegin == 0) ? i : iLastWordBegin - 1; } else { *iEndVal = i; } done: return r; } /* must be positioned on first char of name, returns index * of end of name. * Note: ArcSight violates the CEF spec ifself: they generate * leading underscores in their extension names, which are * definitely not alphanumeric. We still accept them... * They also seem to use dots. */ static int cefParseName(npb_t *const npb, size_t *const __restrict__ i) { int r = 0; while(*i < npb->strLen && npb->str[*i] != '=') { if(!(isalnum(npb->str[*i]) || npb->str[*i] == '_' || npb->str[*i] == '.')) FAIL(LN_WRONGPARSER); ++(*i); } done: return r; } /* parse CEF extensions. They are basically name=value * pairs with the ugly exception that values may contain * spaces but need NOT to be quoted. Thankfully, at least * names are specified as being alphanumeric without spaces * in them. So we must add a lookahead parser to check if * a word is a name (and thus the begin of a new pair) or * not. This is done by subroutines. */ static int cefParseExtensions(npb_t *const npb, size_t *const __restrict__ offs, json_object *const __restrict__ jroot) { int r = 0; size_t i = *offs; size_t iName, lenName; size_t iValue, lenValue; char *name = NULL; char *value = NULL; while(i < npb->strLen) { while(i < npb->strLen && npb->str[i] == ' ') ++i; iName = i; CHKR(cefParseName(npb, &i)); if(npb->str[i] != '=') FAIL(LN_WRONGPARSER); lenName = i - iName; /* Init if the last value is empty */ lenValue = 0; if(i < npb->strLen){ ++i; /* skip '=' */ iValue = i; CHKR(cefParseExtensionValue(npb, &i)); lenValue = i - iValue; ++i; /* skip past value */ } if(jroot != NULL) { CHKN(name = malloc(sizeof(char) * (lenName + 1))); memcpy(name, npb->str+iName, lenName); name[lenName] = '\0'; CHKN(value = malloc(sizeof(char) * (lenValue + 1))); /* copy value but escape it */ size_t iDst = 0; for(size_t iSrc = 0 ; iSrc < lenValue ; ++iSrc) { if(npb->str[iValue+iSrc] == '\\') { ++iSrc; /* we know the next char must exist! */ switch(npb->str[iValue+iSrc]) { case '=': value[iDst] = '='; break; case 'n': value[iDst] = '\n'; break; case 'r': value[iDst] = '\r'; break; case '\\': value[iDst] = '\\'; break; case '/': value[iDst] = '/'; break; default: break; } } else { value[iDst] = npb->str[iValue+iSrc]; } ++iDst; } value[iDst] = '\0'; json_object *json; CHKN(json = json_object_new_string(value)); json_object_object_add(jroot, name, json); free(name); name = NULL; free(value); value = NULL; } } *offs = npb->strLen; /* this parser consume everything or fails */ done: free(name); free(value); return r; } /* gets a CEF header field. Must be positioned on the * first char after the '|' in front of field. * Note that '|' may be escaped as "\|", which also means * we need to supprot "\\" (see CEF spec for details). * We return the string in *val, if val is non-null. In * that case we allocate memory that the caller must free. * This is necessary because there are potentially escape * sequences inside the string. */ static int cefGetHdrField(npb_t *const npb, size_t *const __restrict__ offs, char **val) { int r = 0; size_t i = *offs; assert(npb->str[i] != '|'); while(i < npb->strLen && npb->str[i] != '|') { if(npb->str[i] == '\\') { ++i; /* skip esc char */ if(npb->str[i] != '\\' && npb->str[i] != '|') FAIL(LN_WRONGPARSER); } ++i; /* scan to next delimiter */ } if(npb->str[i] != '|') FAIL(LN_WRONGPARSER); const size_t iBegin = *offs; /* success, persist */ *offs = i + 1; if(val == NULL) { r = 0; goto done; } const size_t len = i - iBegin; CHKN(*val = malloc(len + 1)); size_t iDst = 0; for(size_t iSrc = 0 ; iSrc < len ; ++iSrc) { if(npb->str[iBegin+iSrc] == '\\') ++iSrc; /* we already checked above that this is OK! */ (*val)[iDst++] = npb->str[iBegin+iSrc]; } (*val)[iDst] = 0; r = 0; done: return r; } /** * Parser for ArcSight Common Event Format (CEF) version 0. * added 2015-05-05 by rgerhards, v1.1.2 */ PARSER_Parse(CEF) size_t i = *offs; char *vendor = NULL; char *product = NULL; char *version = NULL; char *sigID = NULL; char *name = NULL; char *severity = NULL; /* minimum header: "CEF:0|x|x|x|x|x|x|" --> 17 chars */ if(npb->strLen < i + 17 || npb->str[i] != 'C' || npb->str[i+1] != 'E' || npb->str[i+2] != 'F' || npb->str[i+3] != ':' || npb->str[i+4] != '0' || npb->str[i+5] != '|' ) FAIL(LN_WRONGPARSER); i += 6; /* position on '|' */ CHKR(cefGetHdrField(npb, &i, (value == NULL) ? NULL : &vendor)); CHKR(cefGetHdrField(npb, &i, (value == NULL) ? NULL : &product)); CHKR(cefGetHdrField(npb, &i, (value == NULL) ? NULL : &version)); CHKR(cefGetHdrField(npb, &i, (value == NULL) ? NULL : &sigID)); CHKR(cefGetHdrField(npb, &i, (value == NULL) ? NULL : &name)); CHKR(cefGetHdrField(npb, &i, (value == NULL) ? NULL : &severity)); while(i < npb->strLen && npb->str[i] == ' ') /* skip leading SP */ ++i; /* OK, we now know we have a good header. Now, we need * to process extensions. * This time, we do NOT pre-process the extension, but rather * persist them directly to JSON. This is contrary to other * parsers, but as the CEF header is pretty unique, this time * it is extremely unlikely we will get a no-match during * extension processing. Even if so, nothing bad happens, as * the extracted data is discarded. But the regular case saves * us processing time and complexity. The only time when we * cannot directly process it is when the caller asks us not * to persist the data. So this must be handled differently. */ size_t iBeginExtensions = i; CHKR(cefParseExtensions(npb, &i, NULL)); /* success, persist */ *parsed = i - *offs; r = 0; /* success */ if(value != NULL) { CHKN(*value = json_object_new_object()); json_object *json; CHKN(json = json_object_new_string(vendor)); json_object_object_add(*value, "DeviceVendor", json); CHKN(json = json_object_new_string(product)); json_object_object_add(*value, "DeviceProduct", json); CHKN(json = json_object_new_string(version)); json_object_object_add(*value, "DeviceVersion", json); CHKN(json = json_object_new_string(sigID)); json_object_object_add(*value, "SignatureID", json); CHKN(json = json_object_new_string(name)); json_object_object_add(*value, "Name", json); CHKN(json = json_object_new_string(severity)); json_object_object_add(*value, "Severity", json); json_object *jext; CHKN(jext = json_object_new_object()); json_object_object_add(*value, "Extensions", jext); i = iBeginExtensions; cefParseExtensions(npb, &i, jext); } done: if(r != 0 && value != NULL && *value != NULL) { json_object_put(*value); value = NULL; } free(vendor); free(product); free(version); free(sigID); free(name); free(severity); return r; } struct data_CheckpointLEA { char terminator; /* '\0' - do not use */ }; /** * Parser for Checkpoint LEA on-disk format. * added 2015-06-18 by rgerhards, v1.1.2 */ PARSER_Parse(CheckpointLEA) size_t i = *offs; size_t iName, lenName; size_t iValue, lenValue; int foundFields = 0; char *name = NULL; char *val = NULL; struct data_CheckpointLEA *const data = (struct data_CheckpointLEA*) pdata; while(i < npb->strLen) { while(i < npb->strLen && npb->str[i] == ' ') /* skip leading SP */ ++i; if(i == npb->strLen) { /* OK if just trailing space */ if(foundFields == 0) FAIL(LN_WRONGPARSER); break; /* we are done with the loop, all processed */ } else { ++foundFields; } iName = i; /* TODO: do a stricter check? ... but we don't have a spec */ if(i < npb->strLen && npb->str[i] == data->terminator) { break; } while(i < npb->strLen && npb->str[i] != ':') { ++i; } if(i+1 >= npb->strLen || npb->str[i] != ':') { FAIL(LN_WRONGPARSER); } lenName = i - iName; ++i; /* skip ':' */ while(i < npb->strLen && npb->str[i] == ' ') /* skip leading SP */ ++i; iValue = i; while(i < npb->strLen && npb->str[i] != ';') { ++i; } if(i+1 > npb->strLen || npb->str[i] != ';') FAIL(LN_WRONGPARSER); lenValue = i - iValue; ++i; /* skip ';' */ if(value != NULL) { CHKN(name = malloc(sizeof(char) * (lenName + 1))); memcpy(name, npb->str+iName, lenName); name[lenName] = '\0'; CHKN(val = malloc(sizeof(char) * (lenValue + 1))); memcpy(val, npb->str+iValue, lenValue); val[lenValue] = '\0'; if(*value == NULL) CHKN(*value = json_object_new_object()); json_object *json; CHKN(json = json_object_new_string(val)); json_object_object_add(*value, name, json); free(name); name = NULL; free(val); val = NULL; } } /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: free(name); free(val); if(r != 0 && value != NULL && *value != NULL) { json_object_put(*value); value = NULL; } return r; } PARSER_Construct(CheckpointLEA) { int r = 0; struct data_CheckpointLEA *data = (struct data_CheckpointLEA*) calloc(1, sizeof(struct data_CheckpointLEA)); if(json == NULL) goto done; struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(!strcmp(key, "terminator")) { const char *const optval = json_object_get_string(val); if(strlen(optval) != 1) { ln_errprintf(ctx, 0, "terminator must be exactly one character " "but is: '%s'", optval); r = LN_BADCONFIG; goto done; } data->terminator = *optval; } json_object_iter_next(&it); } done: *pdata = data; return r; } PARSER_Destruct(CheckpointLEA) { free(pdata); } /* helper to repeat parser constructor: checks that dot field name * is only present if there is one field inside the "parser" list. * returns 1 if ok, 0 otherwise. */ static int chkNoDupeDotInParserDefs(ln_ctx ctx, struct json_object *parsers) { int r = 1; int nParsers = 0; int nDots = 0; if(json_object_get_type(parsers) == json_type_array) { const int maxparsers = json_object_array_length(parsers); for(int i = 0 ; i < maxparsers ; ++i) { ++nParsers; struct json_object *const parser = json_object_array_get_idx(parsers, i); struct json_object *fname; json_object_object_get_ex(parser, "name", &fname); if(fname != NULL) { if(!strcmp(json_object_get_string(fname), ".")) ++nDots; } } } if(nParsers > 1 && nDots > 0) { ln_errprintf(ctx, 0, "'repeat' parser supports dot name only " "if single parser is used in 'parser' part, invalid " "construct: %s", json_object_get_string(parsers)); r = 0; } return r; } /** * "repeat" special parser. */ PARSER_Parse(Repeat) struct data_Repeat *const data = (struct data_Repeat*) pdata; struct ln_pdag *endNode = NULL; size_t strtoffs = *offs; size_t lastKnownGood = strtoffs; struct json_object *json_arr = NULL; const size_t parsedTo_save = npb->parsedTo; do { struct json_object *parsed_value = json_object_new_object(); r = ln_normalizeRec(npb, data->parser, strtoffs, 1, parsed_value, &endNode); strtoffs = npb->parsedTo; LN_DBGPRINTF(npb->ctx, "repeat parser returns %d, parsed %zu, json: %s", r, npb->parsedTo, json_object_to_json_string(parsed_value)); if(r != 0) { json_object_put(parsed_value); if(data->permitMismatchInParser) { strtoffs = lastKnownGood; /* go back to final match */ LN_DBGPRINTF(npb->ctx, "mismatch in repeat, " "parse ptr back to %zd", strtoffs); goto success; } else { goto done; } } if(json_arr == NULL) { json_arr = json_object_new_array(); } /* check for name=".", which means we need to place the * value only into to array. As we do not have direct * access to the key, we loop over our result as a work- * around. */ struct json_object *toAdd = parsed_value; struct json_object_iterator it = json_object_iter_begin(parsed_value); struct json_object_iterator itEnd = json_object_iter_end(parsed_value); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(key[0] == '.' && key[1] == '\0') { json_object_get(val); /* inc refcount! */ toAdd = val; } json_object_iter_next(&it); } json_object_array_add(json_arr, toAdd); if(toAdd != parsed_value) json_object_put(parsed_value); LN_DBGPRINTF(npb->ctx, "arr: %s", json_object_to_json_string(json_arr)); /* now check if we shall continue */ npb->parsedTo = 0; lastKnownGood = strtoffs; /* record pos in case of fail in while */ r = ln_normalizeRec(npb, data->while_cond, strtoffs, 1, NULL, &endNode); LN_DBGPRINTF(npb->ctx, "repeat while returns %d, parsed %zu", r, npb->parsedTo); if(r == 0) strtoffs = npb->parsedTo; } while(r == 0); success: /* success, persist */ *parsed = strtoffs - *offs; if(value == NULL) { json_object_put(json_arr); } else { *value = json_arr; } npb->parsedTo = parsedTo_save; r = 0; /* success */ done: if(r != 0 && json_arr != NULL) { json_object_put(json_arr); } return r; } PARSER_Construct(Repeat) { int r = 0; struct data_Repeat *data = (struct data_Repeat*) calloc(1, sizeof(struct data_Repeat)); struct ln_pdag *endnode; /* we need this fo ln_pdagAddParser, which updates its param! */ if(json == NULL) goto done; struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(!strcmp(key, "parser")) { if(chkNoDupeDotInParserDefs(ctx, val) != 1) { r = LN_BADCONFIG; goto done; } endnode = data->parser = ln_newPDAG(ctx); json_object_get(val); /* prevent free in pdagAddParser */ CHKR(ln_pdagAddParser(ctx, &endnode, val)); endnode->flags.isTerminal = 1; } else if(!strcmp(key, "while")) { endnode = data->while_cond = ln_newPDAG(ctx); json_object_get(val); /* prevent free in pdagAddParser */ CHKR(ln_pdagAddParser(ctx, &endnode, val)); endnode->flags.isTerminal = 1; } else if(!strcasecmp(key, "option.permitMismatchInParser")) { data->permitMismatchInParser = json_object_get_boolean(val); } else { ln_errprintf(ctx, 0, "invalid param for hexnumber: %s", json_object_to_json_string(val)); } json_object_iter_next(&it); } done: if(data->parser == NULL || data->while_cond == NULL) { ln_errprintf(ctx, 0, "repeat parser needs 'parser','while' parameters"); ln_destructRepeat(ctx, data); r = LN_BADCONFIG; } else { *pdata = data; } return r; } PARSER_Destruct(Repeat) { struct data_Repeat *const data = (struct data_Repeat*) pdata; if(data->parser != NULL) ln_pdagDelete(data->parser); if(data->while_cond != NULL) ln_pdagDelete(data->while_cond); free(pdata); } /* string escaping modes */ #define ST_ESC_NONE 0 #define ST_ESC_BACKSLASH 1 #define ST_ESC_DOUBLE 2 #define ST_ESC_BOTH 3 struct data_String { enum { ST_QUOTE_AUTO = 0, ST_QUOTE_NONE = 1, ST_QUOTE_REQD = 2 } quoteMode; struct { unsigned strip_quotes : 1; unsigned esc_md : 2; } flags; enum { ST_MATCH_EXACT = 0, ST_MATCH_LAZY = 1} matching; char qchar_begin; char qchar_end; char perm_chars[256]; // TODO: make this bit-wise, so we need only 32 bytes }; static inline void stringSetPermittedChar(struct data_String *const data, char c, int val) { #if 0 const int i = (unsigned) c / 8; const int shft = (unsigned) c % 8; const unsigned mask = ~(1 << shft); perm_arr[i] = (perm_arr[i] & (0xff #endif data->perm_chars[(unsigned)c] = val; } static inline int stringIsPermittedChar(struct data_String *const data, char c) { return data->perm_chars[(unsigned char)c]; } static void stringAddPermittedCharArr(struct data_String *const data, const char *const optval) { const size_t nchars = strlen(optval); for(size_t i = 0 ; i < nchars ; ++i) { stringSetPermittedChar(data, optval[i], 1); } } static void stringAddPermittedFromTo(struct data_String *const data, const unsigned char from, const unsigned char to) { assert(from <= to); for(size_t i = from ; i <= to ; ++i) { stringSetPermittedChar(data, (char) i, 1); } } static inline void stringAddPermittedChars(struct data_String *const data, struct json_object *const val) { const char *const optval = json_object_get_string(val); if(optval == NULL) return; stringAddPermittedCharArr(data, optval); } static void stringAddPermittedCharsViaArray(ln_ctx ctx, struct data_String *const data, struct json_object *const arr) { const int nelem = json_object_array_length(arr); for(int i = 0 ; i < nelem ; ++i) { struct json_object *const elem = json_object_array_get_idx(arr, i); struct json_object_iterator it = json_object_iter_begin(elem); struct json_object_iterator itEnd = json_object_iter_end(elem); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(!strcasecmp(key, "chars")) { stringAddPermittedChars(data, val); } else if(!strcasecmp(key, "class")) { const char *const optval = json_object_get_string(val); if(!strcasecmp(optval, "digit")) { stringAddPermittedCharArr(data, "0123456789"); } else if(!strcasecmp(optval, "hexdigit")) { stringAddPermittedCharArr(data, "0123456789aAbBcCdDeEfF"); } else if(!strcasecmp(optval, "alpha")) { stringAddPermittedFromTo(data, 'a', 'z'); stringAddPermittedFromTo(data, 'A', 'Z'); } else if(!strcasecmp(optval, "alnum")) { stringAddPermittedCharArr(data, "0123456789"); stringAddPermittedFromTo(data, 'a', 'z'); stringAddPermittedFromTo(data, 'A', 'Z'); } else { ln_errprintf(ctx, 0, "invalid character class '%s'", optval); } } json_object_iter_next(&it); } } } /** * generic string parser */ PARSER_Parse(String) assert(npb->str != NULL); assert(offs != NULL); assert(parsed != NULL); struct data_String *const data = (struct data_String*) pdata; size_t i = *offs; int bHaveQuotes = 0; int bHadEndQuote = 0; int bHadEscape = 0; if(i == npb->strLen) goto done; if((data->quoteMode == ST_QUOTE_AUTO) && (npb->str[i] == data->qchar_begin)) { bHaveQuotes = 1; ++i; } else if(data->quoteMode == ST_QUOTE_REQD) { if(npb->str[i] == data->qchar_begin) { bHaveQuotes = 1; ++i; } else { goto done; } } /* scan string */ while(i < npb->strLen) { if(bHaveQuotes) { if(npb->str[i] == data->qchar_end) { if(data->flags.esc_md == ST_ESC_DOUBLE || data->flags.esc_md == ST_ESC_BOTH) { /* may be escaped, need to check! */ if(i+1 < npb->strLen && npb->str[i+1] == data->qchar_end) { bHadEscape = 1; ++i; } else { /* not escaped -> terminal */ bHadEndQuote = 1; break; } } else { bHadEndQuote = 1; break; } } } if( npb->str[i] == '\\' && i+1 < npb->strLen && (data->flags.esc_md == ST_ESC_BACKSLASH || data->flags.esc_md == ST_ESC_BOTH) ) { bHadEscape = 1; i++; /* skip esc char */ } /* terminating conditions */ if(!bHaveQuotes && npb->str[i] == ' ') break; if(!stringIsPermittedChar(data, npb->str[i])) break; i++; } if(bHaveQuotes && !bHadEndQuote) goto done; if(i == *offs) goto done; if((i - *offs < 1) || (data->matching == ST_MATCH_EXACT)) { const size_t trmChkIdx = (bHaveQuotes) ? i+1 : i; if(npb->str[trmChkIdx] != ' ' && trmChkIdx != npb->strLen) goto done; } /* success, persist */ *parsed = i - *offs; if(bHadEndQuote) ++(*parsed); /* skip quote */ if(value != NULL) { size_t strt; size_t len; if(bHaveQuotes && data->flags.strip_quotes) { strt = *offs + 1; len = *parsed - 2; /* del begin AND end quote! */ } else { strt = *offs; len = *parsed; } char *const cstr = strndup(npb->str+strt, len); CHKN(cstr); if(bHadEscape) { /* need to post-process string... */ for(size_t j = 0 ; cstr[j] != '\0' ; j++) { if( ( cstr[j] == data->qchar_end && cstr[j+1] == data->qchar_end && (data->flags.esc_md == ST_ESC_DOUBLE || data->flags.esc_md == ST_ESC_BOTH) ) || ( cstr[j] == '\\' && (data->flags.esc_md == ST_ESC_BACKSLASH || data->flags.esc_md == ST_ESC_BOTH) ) ) { /* we need to remove the escape character */ memmove(cstr+j, cstr+j+1, len-j); } } } *value = json_object_new_string(cstr); free(cstr); } r = 0; /* success */ done: return r; } PARSER_Construct(String) { int r = 0; struct data_String *const data = (struct data_String*) calloc(1, sizeof(struct data_String)); data->quoteMode = ST_QUOTE_AUTO; data->flags.strip_quotes = 1; data->flags.esc_md = ST_ESC_BOTH; data->qchar_begin = '"'; data->qchar_end = '"'; data->matching = ST_MATCH_EXACT; memset(data->perm_chars, 0xff, sizeof(data->perm_chars)); struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { const char *key = json_object_iter_peek_name(&it); struct json_object *const val = json_object_iter_peek_value(&it); if(!strcasecmp(key, "quoting.mode")) { const char *const optval = json_object_get_string(val); if(!strcasecmp(optval, "auto")) { data->quoteMode = ST_QUOTE_AUTO; } else if(!strcasecmp(optval, "none")) { data->quoteMode = ST_QUOTE_NONE; } else if(!strcasecmp(optval, "required")) { data->quoteMode = ST_QUOTE_REQD; } else { ln_errprintf(ctx, 0, "invalid quoting.mode for string parser: %s", optval); r = LN_BADCONFIG; goto done; } } else if(!strcasecmp(key, "quoting.escape.mode")) { const char *const optval = json_object_get_string(val); if(!strcasecmp(optval, "none")) { data->flags.esc_md = ST_ESC_NONE; } else if(!strcasecmp(optval, "backslash")) { data->flags.esc_md = ST_ESC_BACKSLASH; } else if(!strcasecmp(optval, "double")) { data->flags.esc_md = ST_ESC_DOUBLE; } else if(!strcasecmp(optval, "both")) { data->flags.esc_md = ST_ESC_BOTH; } else { ln_errprintf(ctx, 0, "invalid quoting.escape.mode for string " "parser: %s", optval); r = LN_BADCONFIG; goto done; } } else if(!strcasecmp(key, "quoting.char.begin")) { const char *const optval = json_object_get_string(val); if(strlen(optval) != 1) { ln_errprintf(ctx, 0, "quoting.char.begin must " "be exactly one character but is: '%s'", optval); r = LN_BADCONFIG; goto done; } data->qchar_begin = *optval; } else if(!strcasecmp(key, "quoting.char.end")) { const char *const optval = json_object_get_string(val); if(strlen(optval) != 1) { ln_errprintf(ctx, 0, "quoting.char.end must " "be exactly one character but is: '%s'", optval); r = LN_BADCONFIG; goto done; } data->qchar_end = *optval; } else if(!strcasecmp(key, "matching.permitted")) { memset(data->perm_chars, 0x00, sizeof(data->perm_chars)); if(json_object_is_type(val, json_type_string)) { stringAddPermittedChars(data, val); } else if(json_object_is_type(val, json_type_array)) { stringAddPermittedCharsViaArray(ctx, data, val); } else { ln_errprintf(ctx, 0, "matching.permitted is invalid " "object type, given as '%s", json_object_to_json_string(val)); } } else if(!strcasecmp(key, "matching.mode")) { const char *const optval = json_object_get_string(val); if(!strcasecmp(optval, "strict")) { data->matching = ST_MATCH_EXACT; } else if(!strcasecmp(optval, "lazy")) { data->matching = ST_MATCH_LAZY; } else { ln_errprintf(ctx, 0, "invalid matching.mode for string " "parser: %s", optval); r = LN_BADCONFIG; goto done; } } else { ln_errprintf(ctx, 0, "invalid param for hexnumber: %s", json_object_to_json_string(val)); } json_object_iter_next(&it); } if(data->quoteMode == ST_QUOTE_NONE) data->flags.esc_md = ST_ESC_NONE; *pdata = data; done: if(r != 0) { free(data); } return r; } PARSER_Destruct(String) { free(pdata); } liblognorm-2.0.8/src/parser.h000066400000000000000000000061221511425433100161210ustar00rootroot00000000000000/* * liblognorm - a fast samples-based log normalization library * Copyright 2010-2015 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef LIBLOGNORM_PARSER_H_INCLUDED #define LIBLOGNORM_PARSER_H_INCLUDED #include "pdag.h" /** * Parser interface * @param[in] str the to-be-parsed string * @param[in] strLen length of the to-be-parsed string * @param[in] offs an offset into the string * @param[out] parsed bytes * @param[out] ptr to json object containing parsed data (can be unused) * if NULL on input, object is NOT persisted * @return 0 on success, something else otherwise */ // TODO #warning check how to handle "value" - does it need to be set to NULL? #define PARSERDEF_NO_DATA(parser) \ int ln_v2_parse##parser(npb_t *npb, size_t *offs, void *const, size_t *parsed, struct json_object **value) #define PARSERDEF(parser) \ int ln_construct##parser(ln_ctx ctx, json_object *const json, void **pdata); \ int ln_v2_parse##parser(npb_t *npb, size_t *offs, void *const, size_t *parsed, struct json_object **value); \ void ln_destruct##parser(ln_ctx ctx, void *const pdata) PARSERDEF(RFC5424Date); PARSERDEF(RFC3164Date); PARSERDEF(Number); PARSERDEF(Float); PARSERDEF(HexNumber); PARSERDEF_NO_DATA(KernelTimestamp); PARSERDEF_NO_DATA(Whitespace); PARSERDEF_NO_DATA(Word); PARSERDEF(StringTo); PARSERDEF_NO_DATA(Alpha); PARSERDEF(Literal); PARSERDEF(CharTo); PARSERDEF(CharSeparated); PARSERDEF(Repeat); PARSERDEF(String); PARSERDEF_NO_DATA(Rest); PARSERDEF_NO_DATA(OpQuotedString); PARSERDEF_NO_DATA(QuotedString); PARSERDEF_NO_DATA(ISODate); PARSERDEF_NO_DATA(Time12hr); PARSERDEF_NO_DATA(Time24hr); PARSERDEF_NO_DATA(Duration); PARSERDEF_NO_DATA(IPv4); PARSERDEF_NO_DATA(IPv6); PARSERDEF_NO_DATA(JSON); PARSERDEF_NO_DATA(CEESyslog); PARSERDEF_NO_DATA(v2IPTables); PARSERDEF_NO_DATA(CiscoInterfaceSpec); PARSERDEF_NO_DATA(MAC48); PARSERDEF_NO_DATA(CEF); PARSERDEF(CheckpointLEA); PARSERDEF(NameValue); #undef PARSERDEF_NO_DATA /* utility functions */ int ln_combineData_Literal(void *const org, void *const add); /* definitions for friends */ struct data_Repeat { ln_pdag *parser; ln_pdag *while_cond; int permitMismatchInParser; }; #endif /* #ifndef LIBLOGNORM_PARSER_H_INCLUDED */ liblognorm-2.0.8/src/pdag.c000066400000000000000000001441741511425433100155450ustar00rootroot00000000000000/** * @file pdag.c * @brief Implementation of the parse dag object. * @class ln_pdag pdag.h *//* * Copyright 2015 by Rainer Gerhards and Adiscon GmbH. * * Released under ASL 2.0. */ #include "config.h" #include #include #include #include #include #include #include #include "liblognorm.h" #include "v1_liblognorm.h" #include "v1_ptree.h" #include "lognorm.h" #include "samp.h" #include "pdag.h" #include "annot.h" #include "internal.h" #include "parser.h" #include "helpers.h" void ln_displayPDAGComponentAlternative(struct ln_pdag *dag, int level); void ln_displayPDAGComponent(struct ln_pdag *dag, int level); #ifdef ADVANCED_STATS uint64_t advstats_parsers_called = 0; uint64_t advstats_parsers_success = 0; int advstats_max_pathlen = 0; int advstats_pathlens[ADVSTATS_MAX_ENTITIES]; int advstats_max_backtracked = 0; int advstats_backtracks[ADVSTATS_MAX_ENTITIES]; int advstats_max_parser_calls = 0; int advstats_parser_calls[ADVSTATS_MAX_ENTITIES]; int advstats_max_lit_parser_calls = 0; int advstats_lit_parser_calls[ADVSTATS_MAX_ENTITIES]; #endif /* parser lookup table * This is a memory- and cache-optimized way of calling parsers. * VERY IMPORTANT: the initialization must be done EXACTLY in the * order of parser IDs (also see comment in pdag.h). * * Rough guideline for assigning priorities: * 0 is highest, 255 lowest. 255 should be reserved for things that * *really* should only be run as last resort --> rest. Also keep in * mind that the user-assigned priority is put in the upper 24 bits, so * parser-specific priorities only count when the user has assigned * no priorities (which is expected to be common) or user-assigned * priorities are equal for some parsers. */ #ifdef ADVANCED_STATS #define PARSER_ENTRY_NO_DATA(identifier, parser, prio) \ { identifier, prio, NULL, ln_v2_parse##parser, NULL, 0, 0 } #define PARSER_ENTRY(identifier, parser, prio) \ { identifier, prio, ln_construct##parser, ln_v2_parse##parser, ln_destruct##parser, 0, 0 } #else #define PARSER_ENTRY_NO_DATA(identifier, parser, prio) \ { identifier, prio, NULL, ln_v2_parse##parser, NULL } #define PARSER_ENTRY(identifier, parser, prio) \ { identifier, prio, ln_construct##parser, ln_v2_parse##parser, ln_destruct##parser } #endif static struct ln_parser_info parser_lookup_table[] = { PARSER_ENTRY("literal", Literal, 4), PARSER_ENTRY("repeat", Repeat, 4), PARSER_ENTRY("date-rfc3164", RFC3164Date, 8), PARSER_ENTRY("date-rfc5424", RFC5424Date, 8), PARSER_ENTRY("number", Number, 16), PARSER_ENTRY("float", Float, 16), PARSER_ENTRY("hexnumber", HexNumber, 16), PARSER_ENTRY_NO_DATA("kernel-timestamp", KernelTimestamp, 16), PARSER_ENTRY_NO_DATA("whitespace", Whitespace, 4), PARSER_ENTRY_NO_DATA("ipv4", IPv4, 4), PARSER_ENTRY_NO_DATA("ipv6", IPv6, 4), PARSER_ENTRY_NO_DATA("word", Word, 32), PARSER_ENTRY_NO_DATA("alpha", Alpha, 32), PARSER_ENTRY_NO_DATA("rest", Rest, 255), PARSER_ENTRY_NO_DATA("op-quoted-string", OpQuotedString, 64), PARSER_ENTRY_NO_DATA("quoted-string", QuotedString, 64), PARSER_ENTRY_NO_DATA("date-iso", ISODate, 8), PARSER_ENTRY_NO_DATA("time-24hr", Time24hr, 8), PARSER_ENTRY_NO_DATA("time-12hr", Time12hr, 8), PARSER_ENTRY_NO_DATA("duration", Duration, 16), PARSER_ENTRY_NO_DATA("cisco-interface-spec", CiscoInterfaceSpec, 4), PARSER_ENTRY_NO_DATA("json", JSON, 4), PARSER_ENTRY_NO_DATA("cee-syslog", CEESyslog, 4), PARSER_ENTRY_NO_DATA("mac48", MAC48, 16), PARSER_ENTRY_NO_DATA("cef", CEF, 4), PARSER_ENTRY_NO_DATA("v2-iptables", v2IPTables, 4), PARSER_ENTRY("name-value-list", NameValue, 8), PARSER_ENTRY("checkpoint-lea", CheckpointLEA, 4), PARSER_ENTRY("string-to", StringTo, 32), PARSER_ENTRY("char-to", CharTo, 32), PARSER_ENTRY("char-sep", CharSeparated, 32), PARSER_ENTRY("string", String, 32) }; #define NPARSERS (sizeof(parser_lookup_table)/sizeof(struct ln_parser_info)) #define DFLT_USR_PARSER_PRIO 30000 /**< default priority if user has not specified it */ static inline const char * parserName(const prsid_t id) { const char *name; if(id == PRS_CUSTOM_TYPE) name = "USER-DEFINED"; else name = parser_lookup_table[id].name; return name; } prsid_t ln_parserName2ID(const char *const __restrict__ name) { unsigned i; for( i = 0 ; i < sizeof(parser_lookup_table) / sizeof(struct ln_parser_info) ; ++i) { if(!strcmp(parser_lookup_table[i].name, name)) { return i; } } return PRS_INVALID; } /* find type pdag in table. If "bAdd" is set, add it if not * already present, a new entry will be added. * Returns NULL on error, ptr to type pdag entry otherwise */ struct ln_type_pdag * ln_pdagFindType(ln_ctx ctx, const char *const __restrict__ name, const int bAdd) { struct ln_type_pdag *td = NULL; int i; LN_DBGPRINTF(ctx, "ln_pdagFindType, name '%s', bAdd: %d, nTypes %d", name, bAdd, ctx->nTypes); for(i = 0 ; i < ctx->nTypes ; ++i) { if(!strcmp(ctx->type_pdags[i].name, name)) { td = ctx->type_pdags + i; goto done; } } if(!bAdd) { LN_DBGPRINTF(ctx, "custom type '%s' not found", name); goto done; } /* type does not yet exist -- create entry */ LN_DBGPRINTF(ctx, "custom type '%s' does not yet exist, adding...", name); struct ln_type_pdag *newarr; newarr = realloc(ctx->type_pdags, sizeof(struct ln_type_pdag) * (ctx->nTypes+1)); if(newarr == NULL) { LN_DBGPRINTF(ctx, "ln_pdagFindTypeAG: alloc newarr failed"); goto done; } ctx->type_pdags = newarr; td = ctx->type_pdags + ctx->nTypes; ++ctx->nTypes; td->name = strdup(name); td->pdag = ln_newPDAG(ctx); done: return td; } /* we clear some multiple times, but as long as we have no loops * (dag!) we have no real issue. */ static void ln_pdagComponentClearVisited(struct ln_pdag *const dag) { dag->flags.visited = 0; for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *prs = dag->parsers+i; ln_pdagComponentClearVisited(prs->node); } } static void ln_pdagClearVisited(ln_ctx ctx) { for(int i = 0 ; i < ctx->nTypes ; ++i) ln_pdagComponentClearVisited(ctx->type_pdags[i].pdag); ln_pdagComponentClearVisited(ctx->pdag); } /** * Process a parser definition. Note that a single definition can potentially * contain many parser instances. * @return parser node ptr or NULL (on error) */ ln_parser_t* ln_newParser(ln_ctx ctx, json_object *prscnf) { ln_parser_t *node = NULL; json_object *json; const char *val; prsid_t prsid; struct ln_type_pdag *custType = NULL; const char *name = NULL; const char *textconf = json_object_to_json_string(prscnf); int assignedPrio = DFLT_USR_PARSER_PRIO; int parserPrio; json_object_object_get_ex(prscnf, "type", &json); if(json == NULL) { ln_errprintf(ctx, 0, "parser type missing in config: %s", json_object_to_json_string(prscnf)); goto done; } val = json_object_get_string(json); if(*val == '@') { prsid = PRS_CUSTOM_TYPE; custType = ln_pdagFindType(ctx, val, 0); parserPrio = 16; /* hopefully relatively specific... */ if(custType == NULL) { ln_errprintf(ctx, 0, "unknown user-defined type '%s'", val); goto done; } } else { prsid = ln_parserName2ID(val); if(prsid == PRS_INVALID) { ln_errprintf(ctx, 0, "invalid field type '%s'", val); goto done; } parserPrio = parser_lookup_table[prsid].prio; } json_object_object_get_ex(prscnf, "name", &json); if(json == NULL || !strcmp(json_object_get_string(json), "-")) { name = NULL; } else { name = strdup(json_object_get_string(json)); } json_object_object_get_ex(prscnf, "priority", &json); if(json != NULL) { assignedPrio = json_object_get_int(json); } LN_DBGPRINTF(ctx, "assigned priority is %d", assignedPrio); /* we need to remove already processed items from the config, so * that we can pass the remaining parameters to the parser. */ json_object_object_del(prscnf, "type"); json_object_object_del(prscnf, "priority"); if(name != NULL) json_object_object_del(prscnf, "name"); /* got all data items */ if((node = calloc(1, sizeof(ln_parser_t))) == NULL) { LN_DBGPRINTF(ctx, "lnNewParser: alloc node failed"); free((void*)name); goto done; } node->node = NULL; node->prio = ((assignedPrio << 8) & 0xffffff00) | (parserPrio & 0xff); node->name = name; node->prsid = prsid; node->conf = strdup(textconf); if(prsid == PRS_CUSTOM_TYPE) { node->custTypeIdx = custType - ctx->type_pdags; } else { if(parser_lookup_table[prsid].construct != NULL) { parser_lookup_table[prsid].construct(ctx, prscnf, &node->parser_data); } } done: return node; } struct ln_pdag* ln_newPDAG(ln_ctx ctx) { struct ln_pdag *dag; if((dag = calloc(1, sizeof(struct ln_pdag))) == NULL) goto done; dag->refcnt = 1; dag->ctx = ctx; ctx->nNodes++; done: return dag; } /* note: we must NOT free the parser itself, because * it is stored inside a parser table (so no single * alloc for the parser!). */ static void pdagDeletePrs(ln_ctx ctx, ln_parser_t *const __restrict__ prs) { // TODO: be careful here: once we move to real DAG from tree, we // cannot simply delete the next node! (refcount? something else?) if(prs->node != NULL) ln_pdagDelete(prs->node); free((void*)prs->name); free((void*)prs->conf); if(prs->parser_data != NULL) parser_lookup_table[prs->prsid].destruct(ctx, prs->parser_data); } void ln_pdagDelete(struct ln_pdag *const __restrict__ pdag) { if(pdag == NULL) goto done; LN_DBGPRINTF(pdag->ctx, "delete %p[%d]: %s", pdag, pdag->refcnt, pdag->rb_id); --pdag->refcnt; if(pdag->refcnt > 0) goto done; if(pdag->tags != NULL) json_object_put(pdag->tags); for(int i = 0 ; i < pdag->nparsers ; ++i) { pdagDeletePrs(pdag->ctx, pdag->parsers+i); } free(pdag->parsers); free((void*)pdag->rb_id); free((void*)pdag->rb_file); free(pdag); done: return; } /** * pdag optimizer step: literal path compaction * * We compress as much as possible and evaluate the path down to * the first non-compressable element. Note that we must NOT * compact those literals that are either terminal nodes OR * contain names so that the literal is to be parsed out. */ static inline int optLitPathCompact(ln_ctx ctx, ln_parser_t *prs) { int r = 0; while(prs != NULL) { /* note the NOT prefix in the condition below! */ if(!( prs->prsid == PRS_LITERAL && prs->name == NULL && prs->node->flags.isTerminal == 0 && prs->node->refcnt == 1 && prs->node->nparsers == 1 /* we need to do some checks on the child as well */ && prs->node->parsers[0].prsid == PRS_LITERAL && prs->node->parsers[0].name == NULL && prs->node->parsers[0].node->refcnt == 1) ) goto done; /* ok, we have two compactable literals in a row, let's compact the nodes */ ln_parser_t *child_prs = prs->node->parsers; LN_DBGPRINTF(ctx, "opt path compact: add %p to %p", child_prs, prs); CHKR(ln_combineData_Literal(prs->parser_data, child_prs->parser_data)); ln_pdag *const node_del = prs->node; prs->node = child_prs->node; child_prs->node = NULL; /* remove, else this would be destructed! */ ln_pdagDelete(node_del); } done: return r; } static int qsort_parserCmp(const void *v1, const void *v2) { const ln_parser_t *const p1 = (const ln_parser_t *const) v1; const ln_parser_t *const p2 = (const ln_parser_t *const) v2; return p1->prio - p2->prio; } static int ln_pdagComponentOptimize(ln_ctx ctx, struct ln_pdag *const dag) { int r = 0; for(int i = 0 ; i < dag->nparsers ; ++i) { /* TODO: remove when confident enough */ ln_parser_t *prs = dag->parsers+i; LN_DBGPRINTF(ctx, "pre sort, parser %d:%s[%d]", i, prs->name, prs->prio); } /* first sort parsers in priority order */ if(dag->nparsers > 1) { qsort(dag->parsers, dag->nparsers, sizeof(ln_parser_t), qsort_parserCmp); } for(int i = 0 ; i < dag->nparsers ; ++i) { /* TODO: remove when confident enough */ ln_parser_t *prs = dag->parsers+i; LN_DBGPRINTF(ctx, "post sort, parser %d:%s[%d]", i, prs->name, prs->prio); } /* now on to rest of processing */ for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *prs = dag->parsers+i; LN_DBGPRINTF(dag->ctx, "optimizing %p: field %d type '%s', name '%s': '%s':", prs->node, i, parserName(prs->prsid), prs->name, (prs->prsid == PRS_LITERAL) ? ln_DataForDisplayLiteral(dag->ctx, prs->parser_data) : "UNKNOWN"); optLitPathCompact(ctx, prs); ln_pdagComponentOptimize(ctx, prs->node); } return r; } static void deleteComponentID(struct ln_pdag *const __restrict__ dag) { free((void*)dag->rb_id); dag->rb_id = NULL; for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *prs = dag->parsers+i; deleteComponentID(prs->node); } } /* fixes rb_ids for this node as well as it predecessors. * This is required if the ALTERNATIVE parser type is used, * which will create component IDs for each of it's invocations. * As such, we do not only fix the string, but know that all * children also need fixing. We do this be simply deleting * all of their rb_ids, as we know they will be visited again. * Note: if we introduce the same situation by new functionality, * we may need to review this code here as well. Also note * that the component ID will not be 100% correct after our fix, * because that ID could actually be created by two sets of rules. * But this is the best we can do. */ static void fixComponentID(struct ln_pdag *const __restrict__ dag, const char *const new) { char *updated; const char *const curr = dag->rb_id; int i; int len = (int) strlen(curr); for(i = 0 ; i < len ; ++i){ if(curr[i] != new [i]) break; } if(i >= 1 && curr[i-1] == '%') --i; if(asprintf(&updated, "%.*s[%s|%s]", i, curr, curr+i, new+i) == -1) goto done; deleteComponentID(dag); dag->rb_id = updated; done: return; } /** * Assign human-readable identifiers (names) to each node. These are * later used in stats, debug output and wherever else this may make * sense. */ static void ln_pdagComponentSetIDs(ln_ctx ctx, struct ln_pdag *const dag, const char *prefix) { char *id = NULL; if(prefix == NULL) goto done; if(dag->rb_id == NULL) { dag->rb_id = strdup(prefix); } else { LN_DBGPRINTF(ctx, "rb_id already exists - fixing as good as " "possible. This happens with ALTERNATIVE parser. " "old: '%s', new: '%s'", dag->rb_id, prefix); fixComponentID(dag, prefix); LN_DBGPRINTF(ctx, "\"fixed\" rb_id: %s", dag->rb_id); prefix = dag->rb_id; } /* now on to rest of processing */ for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *prs = dag->parsers+i; if(prs->prsid == PRS_LITERAL) { if(prs->name == NULL) { if(asprintf(&id, "%s%s", prefix, ln_DataForDisplayLiteral(dag->ctx, prs->parser_data)) == -1) goto done; } else { if(asprintf(&id, "%s%%%s:%s:%s%%", prefix, prs->name, parserName(prs->prsid), ln_DataForDisplayLiteral(dag->ctx, prs->parser_data)) == -1) goto done; } } else { if(asprintf(&id, "%s%%%s:%s%%", prefix, prs->name ? prs->name : "-", parserName(prs->prsid)) == -1) goto done; } ln_pdagComponentSetIDs(ctx, prs->node, id); free(id); } done: return; } /** * Optimize the pdag. * This includes all components. */ int ln_pdagOptimize(ln_ctx ctx) { int r = 0; for(int i = 0 ; i < ctx->nTypes ; ++i) { LN_DBGPRINTF(ctx, "optimizing component %s\n", ctx->type_pdags[i].name); ln_pdagComponentOptimize(ctx, ctx->type_pdags[i].pdag); ln_pdagComponentSetIDs(ctx, ctx->type_pdags[i].pdag, ""); } LN_DBGPRINTF(ctx, "optimizing main pdag component"); ln_pdagComponentOptimize(ctx, ctx->pdag); LN_DBGPRINTF(ctx, "finished optimizing main pdag component"); ln_pdagComponentSetIDs(ctx, ctx->pdag, ""); LN_DBGPRINTF(ctx, "---AFTER OPTIMIZATION------------------"); ln_displayPDAG(ctx); LN_DBGPRINTF(ctx, "======================================="); return r; } #define LN_INTERN_PDAG_STATS_NPARSERS 100 /* data structure for pdag statistics */ struct pdag_stats { int nodes; int term_nodes; int parsers; int max_nparsers; int nparsers_cnt[LN_INTERN_PDAG_STATS_NPARSERS]; int nparsers_100plus; int *prs_cnt; }; /** * Recursive step of statistics gatherer. */ static int ln_pdagStatsRec(ln_ctx ctx, struct ln_pdag *const dag, struct pdag_stats *const stats) { if(dag->flags.visited) return 0; dag->flags.visited = 1; stats->nodes++; if(dag->flags.isTerminal) stats->term_nodes++; if(dag->nparsers > stats->max_nparsers) stats->max_nparsers = dag->nparsers; if(dag->nparsers >= LN_INTERN_PDAG_STATS_NPARSERS) stats->nparsers_100plus++; else stats->nparsers_cnt[dag->nparsers]++; stats->parsers += dag->nparsers; int max_path = 0; for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *prs = dag->parsers+i; if(prs->prsid != PRS_CUSTOM_TYPE) stats->prs_cnt[prs->prsid]++; const int path_len = ln_pdagStatsRec(ctx, prs->node, stats); if(path_len > max_path) max_path = path_len; } return max_path + 1; } static void ln_pdagStatsExtended(ln_ctx ctx, struct ln_pdag *const dag, FILE *const fp, int level) { char indent[2048]; if(level > 1023) level = 1023; memset(indent, ' ', level * 2); indent[level * 2] = '\0'; if(dag->stats.called > 0) { fprintf(fp, "%u, %u, %s\n", dag->stats.called, dag->stats.backtracked, dag->rb_id); } for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *const prs = dag->parsers+i; if(prs->node->stats.called > 0) { ln_pdagStatsExtended(ctx, prs->node, fp, level+1); } } } /** * Gather pdag statistics for a *specific* pdag. * * Data is sent to given file ptr. */ static void ln_pdagStats(ln_ctx ctx, struct ln_pdag *const dag, FILE *const fp, const int extendedStats) { struct pdag_stats *const stats = calloc(1, sizeof(struct pdag_stats)); stats->prs_cnt = calloc(NPARSERS, sizeof(int)); //ln_pdagClearVisited(ctx); const int longest_path = ln_pdagStatsRec(ctx, dag, stats); fprintf(fp, "nodes.............: %4d\n", stats->nodes); fprintf(fp, "terminal nodes....: %4d\n", stats->term_nodes); fprintf(fp, "parsers entries...: %4d\n", stats->parsers); fprintf(fp, "longest path......: %4d\n", longest_path); fprintf(fp, "Parser Type Counts:\n"); for(prsid_t i = 0 ; i < NPARSERS ; ++i) { if(stats->prs_cnt[i] != 0) fprintf(fp, "\t%20s: %d\n", parserName(i), stats->prs_cnt[i]); } int pp = 0; fprintf(fp, "Parsers per Node:\n"); fprintf(fp, "\tmax:\t%4d\n", stats->max_nparsers); for(int i = 0 ; i < 100 ; ++i) { pp += stats->nparsers_cnt[i]; if(stats->nparsers_cnt[i] != 0) fprintf(fp, "\t%d:\t%4d\n", i, stats->nparsers_cnt[i]); } free(stats->prs_cnt); free(stats); if(extendedStats) { fprintf(fp, "Usage Statistics:\n" "-----------------\n"); fprintf(fp, "called, backtracked, rule\n"); ln_pdagComponentClearVisited(dag); ln_pdagStatsExtended(ctx, dag, fp, 0); } } /** * Gather and output pdag statistics for the full pdag (ctx) * including all disconnected components (type defs). * * Data is sent to given file ptr. */ void ln_fullPdagStats(ln_ctx ctx, FILE *const fp, const int extendedStats) { if(ctx->ptree != NULL) { /* we need to handle the old cruft */ ln_fullPTreeStats(ctx, fp, extendedStats); return; } fprintf(fp, "User-Defined Types\n" "==================\n"); fprintf(fp, "number types: %d\n", ctx->nTypes); for(int i = 0 ; i < ctx->nTypes ; ++i) fprintf(fp, "type: %s\n", ctx->type_pdags[i].name); for(int i = 0 ; i < ctx->nTypes ; ++i) { fprintf(fp, "\n" "type PDAG: %s\n" "----------\n", ctx->type_pdags[i].name); ln_pdagStats(ctx, ctx->type_pdags[i].pdag, fp, extendedStats); } fprintf(fp, "\n" "Main PDAG\n" "=========\n"); ln_pdagStats(ctx, ctx->pdag, fp, extendedStats); #ifdef ADVANCED_STATS const uint64_t parsers_failed = advstats_parsers_called - advstats_parsers_success; fprintf(fp, "\n" "Advanced Runtime Stats\n" "======================\n"); fprintf(fp, "These are actual number from analyzing the control flow " "at runtime.\n"); fprintf(fp, "Note that literal matching is also done via parsers. As such, \n" "it is expected that fail rates increase with the size of the \n" "rule base.\n"); fprintf(fp, "\n"); fprintf(fp, "Parser Calls:\n"); fprintf(fp, "total....: %10" PRIu64 "\n", advstats_parsers_called); fprintf(fp, "succesful: %10" PRIu64 "\n", advstats_parsers_success); fprintf(fp, "failed...: %10" PRIu64 " [%d%%]\n", parsers_failed, (int) ((parsers_failed * 100) / advstats_parsers_called) ); fprintf(fp, "\nIndividual Parser Calls " "(never called parsers are not shown):\n"); for( size_t i = 0 ; i < sizeof(parser_lookup_table) / sizeof(struct ln_parser_info) ; ++i) { if(parser_lookup_table[i].called > 0) { const uint64_t failed = parser_lookup_table[i].called - parser_lookup_table[i].success; fprintf(fp, "%20s: %10" PRIu64 " [%5.2f%%] " "success: %10" PRIu64 " [%5.1f%%] " "fail: %10" PRIu64 " [%5.1f%%]" "\n", parser_lookup_table[i].name, parser_lookup_table[i].called, (float)(parser_lookup_table[i].called * 100) / advstats_parsers_called, parser_lookup_table[i].success, (float)(parser_lookup_table[i].success * 100) / parser_lookup_table[i].called, failed, (float)(failed * 100) / parser_lookup_table[i].called ); } } uint64_t total_len; uint64_t total_cnt; fprintf(fp, "\n"); fprintf(fp, "\n" "Path Length Statistics\n" "----------------------\n" "The regular path length is the number of nodes being visited,\n" "where each node potentially evaluates several parsers. The\n" "parser call statistic is the number of parsers called along\n" "the path. That number is higher, as multiple parsers may be\n" "called at each node. The number of literal parser calls is\n" "given explicitly, as they use almost no time to process.\n" "\n" ); total_len = 0; total_cnt = 0; fprintf(fp, "Path Length\n"); for(int i = 0 ; i < ADVSTATS_MAX_ENTITIES ; ++i) { if(advstats_pathlens[i] > 0 ) { fprintf(fp, "%3d: %d\n", i, advstats_pathlens[i]); total_len += i * advstats_pathlens[i]; total_cnt += advstats_pathlens[i]; } } fprintf(fp, "avg: %f\n", (double) total_len / (double) total_cnt); fprintf(fp, "max: %d\n", advstats_max_pathlen); fprintf(fp, "\n"); total_len = 0; total_cnt = 0; fprintf(fp, "Nbr Backtracked\n"); for(int i = 0 ; i < ADVSTATS_MAX_ENTITIES ; ++i) { if(advstats_backtracks[i] > 0 ) { fprintf(fp, "%3d: %d\n", i, advstats_backtracks[i]); total_len += i * advstats_backtracks[i]; total_cnt += advstats_backtracks[i]; } } fprintf(fp, "avg: %f\n", (double) total_len / (double) total_cnt); fprintf(fp, "max: %d\n", advstats_max_backtracked); fprintf(fp, "\n"); /* we calc some stats while we output */ total_len = 0; total_cnt = 0; fprintf(fp, "Parser Calls\n"); for(int i = 0 ; i < ADVSTATS_MAX_ENTITIES ; ++i) { if(advstats_parser_calls[i] > 0 ) { fprintf(fp, "%3d: %d\n", i, advstats_parser_calls[i]); total_len += i * advstats_parser_calls[i]; total_cnt += advstats_parser_calls[i]; } } fprintf(fp, "avg: %f\n", (double) total_len / (double) total_cnt); fprintf(fp, "max: %d\n", advstats_max_parser_calls); fprintf(fp, "\n"); total_len = 0; total_cnt = 0; fprintf(fp, "LITERAL Parser Calls\n"); for(int i = 0 ; i < ADVSTATS_MAX_ENTITIES ; ++i) { if(advstats_lit_parser_calls[i] > 0 ) { fprintf(fp, "%3d: %d\n", i, advstats_lit_parser_calls[i]); total_len += i * advstats_lit_parser_calls[i]; total_cnt += advstats_lit_parser_calls[i]; } } fprintf(fp, "avg: %f\n", (double) total_len / (double) total_cnt); fprintf(fp, "max: %d\n", advstats_max_lit_parser_calls); fprintf(fp, "\n"); #endif } /** * Check if the provided dag is a leaf. This means that it * does not contain any subdags. * @return 1 if it is a leaf, 0 otherwise */ static inline int isLeaf(struct ln_pdag *dag) { return dag->nparsers == 0 ? 1 : 0; } /** * Add a parser instance to the pdag at the current position. * * @param[in] ctx * @param[in] prscnf json parser config *object* (no array!) * @param[in] pdag current pdag position (to which parser is to be added) * @param[in/out] nextnode contains point to the next node, either * an existing one or one newly created. * * The nextnode parameter permits to use this function to create * multiple parsers alternative parsers with a single run. To do so, * set nextnode=NULL on first call. On successive calls, keep the * value. If a value is present, we will not accept non-identical * parsers which point to different nodes - this will result in an * error. * * IMPORTANT: the caller is responsible to update its pdag pointer * to the nextnode value when he is done adding parsers. * * If a parser of the same type with identical data already exists, * it is "resued", which means the function is effectively used to * walk the path. This is used during parser construction to * navigate to new parts of the pdag. */ static int ln_pdagAddParserInstance(ln_ctx ctx, json_object *const __restrict__ prscnf, struct ln_pdag *const __restrict__ pdag, struct ln_pdag **nextnode) { int r; ln_parser_t *newtab; LN_DBGPRINTF(ctx, "ln_pdagAddParserInstance: %s, nextnode %p", json_object_to_json_string(prscnf), *nextnode); ln_parser_t *const parser = ln_newParser(ctx, prscnf); CHKN(parser); LN_DBGPRINTF(ctx, "pdag: %p, parser %p", pdag, parser); /* check if we already have this parser, if so, merge */ int i; for(i = 0 ; i < pdag->nparsers ; ++i) { LN_DBGPRINTF(ctx, "parser comparison:\n%s\n%s", pdag->parsers[i].conf, parser->conf); if( pdag->parsers[i].prsid == parser->prsid && !strcmp(pdag->parsers[i].conf, parser->conf)) { // FIXME: the current ->conf object is depending on // the order of json elements. We should do a JSON // comparison (a bit more complex). For now, it // works like we do it now. // FIXME: if nextnode is set, check we can actually combine, // else err out *nextnode = pdag->parsers[i].node; r = 0; LN_DBGPRINTF(ctx, "merging with pdag %p", pdag); pdagDeletePrs(ctx, parser); /* no need for data items */ goto done; } } /* if we reach this point, we have a new parser type */ if(*nextnode == NULL) { CHKN(*nextnode = ln_newPDAG(ctx)); /* we need a new node */ } else { (*nextnode)->refcnt++; } parser->node = *nextnode; newtab = realloc(pdag->parsers, (pdag->nparsers+1) * sizeof(ln_parser_t)); CHKN(newtab); pdag->parsers = newtab; memcpy(pdag->parsers+pdag->nparsers, parser, sizeof(ln_parser_t)); pdag->nparsers++; r = 0; done: free(parser); return r; } static int ln_pdagAddParserInternal(ln_ctx ctx, struct ln_pdag **pdag, const int mode, json_object *const prscnf, struct ln_pdag **nextnode); /** * add parsers to current pdag. This is used * to add parsers stored in an array. The mode specifies * how parsers shall be added. */ #define PRS_ADD_MODE_SEQ 0 #define PRS_ADD_MODE_ALTERNATIVE 1 static int ln_pdagAddParsers(ln_ctx ctx, json_object *const prscnf, const int mode, struct ln_pdag **pdag, struct ln_pdag **p_nextnode) { int r = LN_BADCONFIG; struct ln_pdag *dag = *pdag; struct ln_pdag *nextnode = *p_nextnode; const int lenarr = json_object_array_length(prscnf); for(int i = 0 ; i < lenarr ; ++i) { struct json_object *const curr_prscnf = json_object_array_get_idx(prscnf, i); LN_DBGPRINTF(ctx, "parser %d: %s", i, json_object_to_json_string(curr_prscnf)); if(json_object_get_type(curr_prscnf) == json_type_array) { struct ln_pdag *local_dag = dag; CHKR(ln_pdagAddParserInternal(ctx, &local_dag, mode, curr_prscnf, &nextnode)); if(mode == PRS_ADD_MODE_SEQ) { dag = local_dag; } } else { CHKR(ln_pdagAddParserInstance(ctx, curr_prscnf, dag, &nextnode)); } if(mode == PRS_ADD_MODE_SEQ) { dag = nextnode; *p_nextnode = nextnode; nextnode = NULL; } } if(mode != PRS_ADD_MODE_SEQ) dag = nextnode; *pdag = dag; r = 0; done: return r; } /* add a json parser config object. Note that this object may contain * multiple parser instances. Additionally, moves the pdag object to * the next node, which is either newly created or previously existed. */ static int ln_pdagAddParserInternal(ln_ctx ctx, struct ln_pdag **pdag, const int mode, json_object *const prscnf, struct ln_pdag **nextnode) { int r = LN_BADCONFIG; struct ln_pdag *dag = *pdag; LN_DBGPRINTF(ctx, "ln_pdagAddParserInternal: %s", json_object_to_json_string(prscnf)); if(json_object_get_type(prscnf) == json_type_object) { /* check for special types we need to handle here */ struct json_object *json; json_object_object_get_ex(prscnf, "type", &json); const char *const ftype = json_object_get_string(json); if(!strcmp(ftype, "alternative")) { json_object_object_get_ex(prscnf, "parser", &json); if(json_object_get_type(json) != json_type_array) { ln_errprintf(ctx, 0, "alternative type needs array of parsers. " "Object: '%s', type is %s", json_object_to_json_string(prscnf), json_type_to_name(json_object_get_type(json))); goto done; } CHKR(ln_pdagAddParsers(ctx, json, PRS_ADD_MODE_ALTERNATIVE, &dag, nextnode)); } else { CHKR(ln_pdagAddParserInstance(ctx, prscnf, dag, nextnode)); if(mode == PRS_ADD_MODE_SEQ) dag = *nextnode; } } else if(json_object_get_type(prscnf) == json_type_array) { CHKR(ln_pdagAddParsers(ctx, prscnf, PRS_ADD_MODE_SEQ, &dag, nextnode)); } else { ln_errprintf(ctx, 0, "bug: prscnf object of wrong type. Object: '%s'", json_object_to_json_string(prscnf)); goto done; } *pdag = dag; done: return r; } /* add a json parser config object. Note that this object may contain * multiple parser instances. Additionally, moves the pdag object to * the next node, which is either newly created or previously existed. */ int ln_pdagAddParser(ln_ctx ctx, struct ln_pdag **pdag, json_object *const prscnf) { struct ln_pdag *nextnode = NULL; int r = ln_pdagAddParserInternal(ctx, pdag, PRS_ADD_MODE_SEQ, prscnf, &nextnode); json_object_put(prscnf); return r; } void ln_displayPDAGComponent(struct ln_pdag *dag, int level) { char indent[2048]; if(level > 1023) level = 1023; memset(indent, ' ', level * 2); indent[level * 2] = '\0'; LN_DBGPRINTF(dag->ctx, "%ssubDAG%s %p (children: %d parsers, ref %d) [called %u, backtracked %u]", indent, dag->flags.isTerminal ? " [TERM]" : "", dag, dag->nparsers, dag->refcnt, dag->stats.called, dag->stats.backtracked); for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *const prs = dag->parsers+i; LN_DBGPRINTF(dag->ctx, "%sfield type '%s', name '%s': '%s': called %u", indent, parserName(prs->prsid), dag->parsers[i].name, (prs->prsid == PRS_LITERAL) ? ln_DataForDisplayLiteral(dag->ctx, prs->parser_data) : "UNKNOWN", dag->parsers[i].node->stats.called); } for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *const prs = dag->parsers+i; LN_DBGPRINTF(dag->ctx, "%sfield type '%s', name '%s': '%s':", indent, parserName(prs->prsid), dag->parsers[i].name, (prs->prsid == PRS_LITERAL) ? ln_DataForDisplayLiteral(dag->ctx, prs->parser_data) : "UNKNOWN"); if(prs->prsid == PRS_REPEAT) { struct data_Repeat *const data = (struct data_Repeat*) prs->parser_data; LN_DBGPRINTF(dag->ctx, "%sparser:", indent); ln_displayPDAGComponent(data->parser, level + 1); LN_DBGPRINTF(dag->ctx, "%swhile:", indent); ln_displayPDAGComponent(data->while_cond, level + 1); LN_DBGPRINTF(dag->ctx, "%send repeat def", indent); } ln_displayPDAGComponent(dag->parsers[i].node, level + 1); } } void ln_displayPDAGComponentAlternative(struct ln_pdag *dag, int level) { char indent[2048]; if(level > 1023) level = 1023; memset(indent, ' ', level * 2); indent[level * 2] = '\0'; LN_DBGPRINTF(dag->ctx, "%s%p[ref %d]: %s", indent, dag, dag->refcnt, dag->rb_id); for(int i = 0 ; i < dag->nparsers ; ++i) { ln_displayPDAGComponentAlternative(dag->parsers[i].node, level + 1); } } /* developer debug aid, to be used for example as follows: * LN_DBGPRINTF(dag->ctx, "---------------------------------------"); * ln_displayPDAG(dag); * LN_DBGPRINTF(dag->ctx, "======================================="); */ void ln_displayPDAG(ln_ctx ctx) { ln_pdagClearVisited(ctx); for(int i = 0 ; i < ctx->nTypes ; ++i) { LN_DBGPRINTF(ctx, "COMPONENT: %s", ctx->type_pdags[i].name); ln_displayPDAGComponent(ctx->type_pdags[i].pdag, 0); } LN_DBGPRINTF(ctx, "MAIN COMPONENT:"); ln_displayPDAGComponent(ctx->pdag, 0); LN_DBGPRINTF(ctx, "MAIN COMPONENT (alternative):"); ln_displayPDAGComponentAlternative(ctx->pdag, 0); } /* the following is a quick hack, which should be moved to the * string class. */ static inline void dotAddPtr(es_str_t **str, void *p) { char buf[64]; int i; i = snprintf(buf, sizeof(buf), "l%p", p); es_addBuf(str, buf, i); } struct data_Literal { const char *lit; }; // TODO remove when this hack is no longer needed /** * recursive handler for DOT graph generator. */ static void ln_genDotPDAGGraphRec(struct ln_pdag *dag, es_str_t **str) { char s_refcnt[16]; LN_DBGPRINTF(dag->ctx, "in dot: %p, visited %d", dag, (int) dag->flags.visited); if(dag->flags.visited) return; /* already processed this subpart */ dag->flags.visited = 1; dotAddPtr(str, dag); snprintf(s_refcnt, sizeof(s_refcnt), "%d", dag->refcnt); s_refcnt[sizeof(s_refcnt)-1] = '\0'; es_addBufConstcstr(str, " [ label=\""); es_addBuf(str, s_refcnt, strlen(s_refcnt)); es_addBufConstcstr(str, "\""); if(isLeaf(dag)) { es_addBufConstcstr(str, " style=\"bold\""); } es_addBufConstcstr(str, "]\n"); /* display field subdags */ for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *const prs = dag->parsers+i; dotAddPtr(str, dag); es_addBufConstcstr(str, " -> "); dotAddPtr(str, prs->node); es_addBufConstcstr(str, " [label=\""); es_addBuf(str, parserName(prs->prsid), strlen(parserName(prs->prsid))); es_addBufConstcstr(str, ":"); //es_addStr(str, node->name); if(prs->prsid == PRS_LITERAL) { for(const char *p = ((struct data_Literal*)prs->parser_data)->lit ; *p ; ++p) { // TODO: handle! if(*p == '\\') //es_addChar(str, '\\'); if(*p != '\\' && *p != '"') es_addChar(str, *p); } } es_addBufConstcstr(str, "\""); es_addBufConstcstr(str, " style=\"dotted\"]\n"); ln_genDotPDAGGraphRec(prs->node, str); } } void ln_genDotPDAGGraph(struct ln_pdag *dag, es_str_t **str) { ln_pdagClearVisited(dag->ctx); es_addBufConstcstr(str, "digraph pdag {\n"); ln_genDotPDAGGraphRec(dag, str); es_addBufConstcstr(str, "}\n"); } /** * recursive handler for statistics DOT graph generator. */ static void ln_genStatsDotPDAGGraphRec(struct ln_pdag *dag, FILE *const __restrict__ fp) { if(dag->flags.visited) return; /* already processed this subpart */ dag->flags.visited = 1; fprintf(fp, "l%p [ label=\"%u:%u\"", dag, dag->stats.called, dag->stats.backtracked); if(isLeaf(dag)) { fprintf(fp, " style=\"bold\""); } fprintf(fp, "]\n"); /* display field subdags */ for(int i = 0 ; i < dag->nparsers ; ++i) { ln_parser_t *const prs = dag->parsers+i; if(prs->node->stats.called == 0) continue; fprintf(fp, "l%p -> l%p [label=\"", dag, prs->node); if(prs->prsid == PRS_LITERAL) { for(const char *p = ((struct data_Literal*)prs->parser_data)->lit ; *p ; ++p) { if(*p != '\\' && *p != '"') fputc(*p, fp); } } else { fprintf(fp, "%s", parserName(prs->prsid)); } fprintf(fp, "\" style=\"dotted\"]\n"); ln_genStatsDotPDAGGraphRec(prs->node, fp); } } static void ln_genStatsDotPDAGGraph(struct ln_pdag *dag, FILE *const fp) { ln_pdagClearVisited(dag->ctx); fprintf(fp, "digraph pdag {\n"); ln_genStatsDotPDAGGraphRec(dag, fp); fprintf(fp, "}\n"); } void ln_fullPDagStatsDOT(ln_ctx ctx, FILE *const fp) { ln_genStatsDotPDAGGraph(ctx->pdag, fp); } static inline int addOriginalMsg(const char *str, const size_t strLen, struct json_object *const json) { int r = 1; struct json_object *value; value = json_object_new_string_len(str, strLen); if (value == NULL) { goto done; } json_object_object_add(json, ORIGINAL_MSG_KEY, value); r = 0; done: return r; } static char * strrev(char *const __restrict__ str) { char ch; size_t i = strlen(str)-1,j=0; while(i>j) { ch = str[i]; str[i]= str[j]; str[j] = ch; i--; j++; } return str; } /* note: "originalmsg" is NOT added as metadata in order to keep * backwards compatible. */ static inline void addRuleMetadata(npb_t *const __restrict__ npb, struct json_object *const json, struct ln_pdag *const __restrict__ endNode) { ln_ctx ctx = npb->ctx; struct json_object *meta = NULL; struct json_object *meta_rule = NULL; struct json_object *value; if(ctx->opts & LN_CTXOPT_ADD_RULE) { /* matching rule mockup */ if(meta_rule == NULL) meta_rule = json_object_new_object(); char *cstr = strrev(es_str2cstr(npb->rule, NULL)); json_object_object_add(meta_rule, RULE_MOCKUP_KEY, json_object_new_string(cstr)); free(cstr); } if(ctx->opts & LN_CTXOPT_ADD_RULE_LOCATION) { if(meta_rule == NULL) meta_rule = json_object_new_object(); struct json_object *const location = json_object_new_object(); value = json_object_new_string(endNode->rb_file); json_object_object_add(location, "file", value); value = json_object_new_int((int)endNode->rb_lineno); json_object_object_add(location, "line", value); json_object_object_add(meta_rule, RULE_LOCATION_KEY, location); } if(meta_rule != NULL) { if(meta == NULL) meta = json_object_new_object(); json_object_object_add(meta, META_RULE_KEY, meta_rule); } #ifdef ADVANCED_STATS /* complete execution path */ if(ctx->opts & LN_CTXOPT_ADD_EXEC_PATH) { if(meta == NULL) meta = json_object_new_object(); char hdr[128]; const size_t lenhdr = snprintf(hdr, sizeof(hdr), "[PATHLEN:%d, PARSER CALLS gen:%d, literal:%d]", npb->astats.pathlen, npb->astats.parser_calls, npb->astats.lit_parser_calls); es_addBuf(&npb->astats.exec_path, hdr, lenhdr); char * cstr = es_str2cstr(npb->astats.exec_path, NULL); value = json_object_new_string(cstr); if (value != NULL) { json_object_object_add(meta, EXEC_PATH_KEY, value); } free(cstr); } #endif if(meta != NULL) json_object_object_add(json, META_KEY, meta); } /** * add unparsed string to event. */ static inline int addUnparsedField(const char *str, const size_t strLen, const size_t offs, struct json_object *json) { int r = 1; struct json_object *value; CHKR(addOriginalMsg(str, strLen, json)); value = json_object_new_string(str + offs); if (value == NULL) { goto done; } json_object_object_add(json, UNPARSED_DATA_KEY, value); r = 0; done: return r; } /* Do some fixup to the json that we cannot do on a lower layer */ static int fixJSON(struct ln_pdag *dag, struct json_object **value, struct json_object *json, const ln_parser_t *const prs) { int r = LN_WRONGPARSER; if(prs->name == NULL) { if (*value != NULL) { /* Free the unneeded value */ json_object_put(*value); } } else if(prs->name[0] == '.' && prs->name[1] == '\0') { if(json_object_get_type(*value) == json_type_object) { struct json_object_iterator it = json_object_iter_begin(*value); struct json_object_iterator itEnd = json_object_iter_end(*value); while (!json_object_iter_equal(&it, &itEnd)) { struct json_object *const val = json_object_iter_peek_value(&it); json_object_get(val); json_object_object_add(json, json_object_iter_peek_name(&it), val); json_object_iter_next(&it); } json_object_put(*value); } else { LN_DBGPRINTF(dag->ctx, "field name is '.', but json type is %s", json_type_to_name(json_object_get_type(*value))); json_object_object_add_ex(json, prs->name, *value, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); } } else { int isDotDot = 0; struct json_object *valDotDot = NULL; if(json_object_get_type(*value) == json_type_object) { /* TODO: this needs to be sped up by just checking the first * member and ensuring there is only one member. This requires * extensions to libfastjson. */ int nSubobj = 0; struct json_object_iterator it = json_object_iter_begin(*value); struct json_object_iterator itEnd = json_object_iter_end(*value); while (!json_object_iter_equal(&it, &itEnd)) { ++nSubobj; const char *key = json_object_iter_peek_name(&it); if(key[0] == '.' && key[1] == '.' && key[2] == '\0') { isDotDot = 1; valDotDot = json_object_iter_peek_value(&it); } else { isDotDot = 0; } json_object_iter_next(&it); } if(nSubobj != 1) isDotDot = 0; } if(isDotDot) { LN_DBGPRINTF(dag->ctx, "subordinate field name is '..', combining"); json_object_get(valDotDot); json_object_put(*value); json_object_object_add_ex(json, prs->name, valDotDot, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); } else { json_object_object_add_ex(json, prs->name, *value, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); } } r = 0; return r; } // TODO: streamline prototype when done with changes static int tryParser(npb_t *const __restrict__ npb, struct ln_pdag *dag, size_t *offs, size_t *const __restrict__ pParsed, struct json_object **value, const ln_parser_t *const prs ) { int r; struct ln_pdag *endNode = NULL; size_t parsedTo = npb->parsedTo; # ifdef ADVANCED_STATS char hdr[16]; const size_t lenhdr = snprintf(hdr, sizeof(hdr), "%d:", npb->astats.recursion_level); es_addBuf(&npb->astats.exec_path, hdr, lenhdr); if(prs->prsid == PRS_LITERAL) { es_addChar(&npb->astats.exec_path, '\''); es_addBuf(&npb->astats.exec_path, ln_DataForDisplayLiteral(dag->ctx, prs->parser_data), strlen(ln_DataForDisplayLiteral(dag->ctx, prs->parser_data)) ); es_addChar(&npb->astats.exec_path, '\''); } else if(parser_lookup_table[prs->prsid].parser == ln_v2_parseCharTo) { es_addBuf(&npb->astats.exec_path, ln_DataForDisplayCharTo(dag->ctx, prs->parser_data), strlen(ln_DataForDisplayCharTo(dag->ctx, prs->parser_data)) ); } else { es_addBuf(&npb->astats.exec_path, parserName(prs->prsid), strlen(parserName(prs->prsid)) ); } es_addChar(&npb->astats.exec_path, ','); # endif if(prs->prsid == PRS_CUSTOM_TYPE) { if(*value == NULL) *value = json_object_new_object(); LN_DBGPRINTF(dag->ctx, "calling custom parser '%s'", dag->ctx->type_pdags[prs->custTypeIdx].name); r = ln_normalizeRec(npb, dag->ctx->type_pdags[prs->custTypeIdx].pdag, *offs, 1, *value, &endNode); LN_DBGPRINTF(dag->ctx, "called CUSTOM PARSER '%s', result %d, " "offs %zd, *pParsed %zd", dag->ctx->type_pdags[prs->custTypeIdx].name, r, *offs, *pParsed); *pParsed = npb->parsedTo - *offs; #ifdef ADVANCED_STATS es_addBuf(&npb->astats.exec_path, hdr, lenhdr); es_addBuf(&npb->astats.exec_path, "[R:USR],", 8); #endif } else { r = parser_lookup_table[prs->prsid].parser(npb, offs, prs->parser_data, pParsed, (prs->name == NULL) ? NULL : value); } LN_DBGPRINTF(npb->ctx, "parser lookup returns %d, pParsed %zu", r, *pParsed); npb->parsedTo = parsedTo; #ifdef ADVANCED_STATS ++advstats_parsers_called; ++npb->astats.parser_calls; if(prs->prsid == PRS_LITERAL) ++npb->astats.lit_parser_calls; if(r == 0) ++advstats_parsers_success; if(prs->prsid != PRS_CUSTOM_TYPE) { ++parser_lookup_table[prs->prsid].called; if(r == 0) ++parser_lookup_table[prs->prsid].success; } #endif return r; } static void add_str_reversed(npb_t *const __restrict__ npb, const char *const __restrict__ str, const size_t len) { ssize_t i; for(i = len - 1 ; i >= 0 ; --i) { es_addChar(&npb->rule, str[i]); } } /* Add the current parser to the mockup rule. * Note: we add reversed strings, because we can call this * function effectively only when walking upwards the tree. * This means deepest entries come first. We solve this somewhat * elegantly by reversion strings, and then reversion the string * once more when we emit it, so that we get the right order. */ static inline void add_rule_to_mockup(npb_t *const __restrict__ npb, const ln_parser_t *const __restrict__ prs) { if(prs->prsid == PRS_LITERAL) { const char *const val = ln_DataForDisplayLiteral(npb->ctx, prs->parser_data); add_str_reversed(npb, val, strlen(val)); } else { /* note: name/value order must also be reversed! */ es_addChar(&npb->rule, '%'); add_str_reversed(npb, parserName(prs->prsid), strlen(parserName(prs->prsid)) ); es_addChar(&npb->rule, ':'); if(prs->name == NULL) { es_addChar(&npb->rule, '-'); } else { add_str_reversed(npb, prs->name, strlen(prs->name)); } es_addChar(&npb->rule, '%'); } } /** * Recursive step of the normalizer. It walks the parse dag and calls itself * recursively when this is appropriate. It also implements backtracking in * those (hopefully rare) cases where it is required. * * @param[in] dag current tree to process * @param[in] string string to be matched against (the to-be-normalized data) * @param[in] strLen length of the to-be-matched string * @param[in] offs start position in input data * @param[out] pPrasedTo ptr to position up to which the parsing succeed in max * @param[in/out] json ... that is being created during normalization * @param[out] endNode if a match was found, this is the matching node (undefined otherwise) * * @return regular liblognorm error code (0->OK, something else->error) * TODO: can we use parameter block to prevent pushing params to the stack? */ int ln_normalizeRec(npb_t *const __restrict__ npb, struct ln_pdag *dag, const size_t offs, const int bPartialMatch, struct json_object *json, struct ln_pdag **endNode ) { int r = LN_WRONGPARSER; int localR; size_t i; size_t iprs; size_t parsedTo = npb->parsedTo; size_t parsed = 0; struct json_object *value; LN_DBGPRINTF(dag->ctx, "%zu: enter parser, dag node %p, json %p", offs, dag, json); ++dag->stats.called; #ifdef ADVANCED_STATS ++npb->astats.pathlen; ++npb->astats.recursion_level; #endif /* now try the parsers */ for(iprs = 0 ; iprs < dag->nparsers && r != 0 ; ++iprs) { const ln_parser_t *const prs = dag->parsers + iprs; if(dag->ctx->debug) { LN_DBGPRINTF(dag->ctx, "%zu/%d:trying '%s' parser for field '%s', " "data '%s'", offs, bPartialMatch, parserName(prs->prsid), prs->name, (prs->prsid == PRS_LITERAL) ? ln_DataForDisplayLiteral(dag->ctx, prs->parser_data) : "UNKNOWN"); } i = offs; value = NULL; localR = tryParser(npb, dag, &i, &parsed, &value, prs); if(localR == 0) { parsedTo = i + parsed; /* potential hit, need to verify */ LN_DBGPRINTF(dag->ctx, "%zu: potential hit, trying subtree %p", offs, prs->node); r = ln_normalizeRec(npb, prs->node, parsedTo, bPartialMatch, json, endNode); LN_DBGPRINTF(dag->ctx, "%zu: subtree returns %d, parsedTo %zu", offs, r, parsedTo); if(r == 0) { LN_DBGPRINTF(dag->ctx, "%zu: parser matches at %zu", offs, i); CHKR(fixJSON(dag, &value, json, prs)); if(npb->ctx->opts & LN_CTXOPT_ADD_RULE) { add_rule_to_mockup(npb, prs); } } else { ++dag->stats.backtracked; #ifdef ADVANCED_STATS ++npb->astats.backtracked; es_addBuf(&npb->astats.exec_path, "[B]", 3); #endif LN_DBGPRINTF(dag->ctx, "%zu nonmatch, backtracking required, parsed to=%zu", offs, parsedTo); if (value != NULL) { /* Free the value if it was created */ json_object_put(value); } } } /* did we have a longer parser --> then update */ if(parsedTo > npb->parsedTo) npb->parsedTo = parsedTo; LN_DBGPRINTF(dag->ctx, "parsedTo %zu, *pParsedTo %zu", parsedTo, npb->parsedTo); } LN_DBGPRINTF(dag->ctx, "offs %zu, strLen %zu, isTerm %d", offs, npb->strLen, dag->flags.isTerminal); if(dag->flags.isTerminal && (offs == npb->strLen || bPartialMatch)) { *endNode = dag; r = 0; goto done; } done: LN_DBGPRINTF(dag->ctx, "%zu returns %d, pParsedTo %zu, parsedTo %zu", offs, r, npb->parsedTo, parsedTo); # ifdef ADVANCED_STATS --npb->astats.recursion_level; # endif return r; } int ln_normalize(ln_ctx ctx, const char *str, const size_t strLen, struct json_object **json_p) { int r; struct ln_pdag *endNode = NULL; /* old cruft */ if(ctx->version == 1) { r = ln_v1_normalize(ctx, str, strLen, json_p); goto done; } /* end old cruft */ npb_t npb; memset(&npb, 0, sizeof(npb)); npb.ctx = ctx; npb.str = str; npb.strLen = strLen; if(ctx->opts & LN_CTXOPT_ADD_RULE) { npb.rule = es_newStr(1024); } # ifdef ADVANCED_STATS npb.astats.exec_path = es_newStr(1024); # endif if(*json_p == NULL) { CHKN(*json_p = json_object_new_object()); } r = ln_normalizeRec(&npb, ctx->pdag, 0, 0, *json_p, &endNode); if(ctx->debug) { if(r == 0) { LN_DBGPRINTF(ctx, "final result for normalizer: parsedTo %zu, endNode %p, " "isTerminal %d, tagbucket %p", npb.parsedTo, endNode, endNode->flags.isTerminal, endNode->tags); } else { LN_DBGPRINTF(ctx, "final result for normalizer: parsedTo %zu, endNode %p", npb.parsedTo, endNode); } } LN_DBGPRINTF(ctx, "DONE, final return is %d", r); if(r == 0 && endNode->flags.isTerminal) { /* success, finalize event */ if(endNode->tags != NULL) { /* add tags to an event */ json_object_get(endNode->tags); json_object_object_add(*json_p, "event.tags", endNode->tags); CHKR(ln_annotate(ctx, *json_p, endNode->tags)); } if(ctx->opts & LN_CTXOPT_ADD_ORIGINALMSG) { /* originalmsg must be kept outside of metadata for * backward compatibility reasons. */ json_object_object_add(*json_p, ORIGINAL_MSG_KEY, json_object_new_string_len(str, strLen)); } addRuleMetadata(&npb, *json_p, endNode); r = 0; } else { addUnparsedField(str, strLen, npb.parsedTo, *json_p); } if(ctx->opts & LN_CTXOPT_ADD_RULE) { es_deleteStr(npb.rule); } #ifdef ADVANCED_STATS if(r != 0) es_addBuf(&npb.astats.exec_path, "[FAILED]", 8); else if(!endNode->flags.isTerminal) es_addBuf(&npb.astats.exec_path, "[FAILED:NON-TERMINAL]", 21); if(npb.astats.pathlen < ADVSTATS_MAX_ENTITIES) advstats_pathlens[npb.astats.pathlen]++; if(npb.astats.pathlen > advstats_max_pathlen) { advstats_max_pathlen = npb.astats.pathlen; } if(npb.astats.backtracked < ADVSTATS_MAX_ENTITIES) advstats_backtracks[npb.astats.backtracked]++; if(npb.astats.backtracked > advstats_max_backtracked) { advstats_max_backtracked = npb.astats.backtracked; } /* parser calls */ if(npb.astats.parser_calls < ADVSTATS_MAX_ENTITIES) advstats_parser_calls[npb.astats.parser_calls]++; if(npb.astats.parser_calls > advstats_max_parser_calls) { advstats_max_parser_calls = npb.astats.parser_calls; } if(npb.astats.lit_parser_calls < ADVSTATS_MAX_ENTITIES) advstats_lit_parser_calls[npb.astats.lit_parser_calls]++; if(npb.astats.lit_parser_calls > advstats_max_lit_parser_calls) { advstats_max_lit_parser_calls = npb.astats.lit_parser_calls; } es_deleteStr(npb.astats.exec_path); #endif done: return r; } liblognorm-2.0.8/src/pdag.h000066400000000000000000000203261511425433100155420ustar00rootroot00000000000000/** * @file pdag.h * @brief The parse DAG object. * @class ln_pdag pdag.h *//* * Copyright 2015 by Rainer Gerhards and Adiscon GmbH. * * Released under ASL 2.0. */ #ifndef LIBLOGNORM_PDAG_H_INCLUDED #define LIBLOGNORM_PDAG_H_INCLUDED #include #include #include #define META_KEY "metadata" #define ORIGINAL_MSG_KEY "originalmsg" #define UNPARSED_DATA_KEY "unparsed-data" #define EXEC_PATH_KEY "exec-path" #define META_RULE_KEY "rule" #define RULE_MOCKUP_KEY "mockup" #define RULE_LOCATION_KEY "location" typedef struct ln_pdag ln_pdag; /**< the parse DAG object */ typedef struct ln_parser_s ln_parser_t; typedef struct npb npb_t; typedef uint8_t prsid_t; struct ln_type_pdag; /** * parser IDs. * * These identfy a parser. VERY IMPORTANT: they must start at zero * and continuously increment. They must exactly match the index * of the respective parser inside the parser lookup table. */ #define PRS_LITERAL 0 #define PRS_REPEAT 1 #if 0 #define PRS_DATE_RFC3164 1 #define PRS_DATE_RFC5424 2 #define PRS_NUMBER 3 #define PRS_FLOAT 4 #define PRS_HEXNUMBER 5 #define PRS_KERNEL_TIMESTAMP 6 #define PRS_WHITESPACE 7 #define PRS_IPV4 8 #define PRS_IPV6 9 #define PRS_WORD 10 #define PRS_ALPHA 11 #define PRS_REST 12 #define PRS_OP_QUOTED_STRING 13 #define PRS_QUOTED_STRING 14 #define PRS_DATE_ISO 15 #define PRS_TIME_24HR 16 #define PRS_TIME_12HR 17 #define PRS_DURATION 18 #define PRS_CISCO_INTERFACE_SPEC 19 #define PRS_NAME_VALUE_LIST 20 #define PRS_JSON 21 #define PRS_CEE_SYSLOG 22 #define PRS_MAC48 23 #define PRS_CEF 24 #define PRS_CHECKPOINT_LEA 25 #define PRS_v2_IPTABLES 26 #define PRS_STRING_TO 27 #define PRS_CHAR_TO 28 #define PRS_CHAR_SEP 29 #endif #define PRS_CUSTOM_TYPE 254 #define PRS_INVALID 255 /* NOTE: current max limit on parser ID is 255, because we use uint8_t * for the prsid_t type (which gains cache performance). If more parsers * come up, the type must be modified. */ /** * object describing a specific parser instance. */ struct ln_parser_s { prsid_t prsid; /**< parser ID (for lookup table) */ ln_pdag *node; /**< node to branch to if parser succeeded */ void *parser_data; /**< opaque data that the field-parser understands */ size_t custTypeIdx; /**< index to custom type, if such is used */ int prio; /**< priority (combination of user- and parser-specific parts) */ const char *name; /**< field name */ const char *conf; /**< configuration as printable json for comparison reasons */ }; struct ln_parser_info { const char *name; /**< parser name as used in rule base */ int prio; /**< parser specific prio in range 0..255 */ int (*construct)(ln_ctx ctx, json_object *const json, void **); int (*parser)(npb_t *npb, size_t*, void *const, size_t*, struct json_object **); /**< parser to use */ void (*destruct)(ln_ctx, void *const); /* note: destructor is only needed if parser data exists */ #ifdef ADVANCED_STATS uint64_t called; uint64_t success; #endif }; /* parse DAG object */ struct ln_pdag { ln_ctx ctx; /**< our context */ // TODO: why do we need it? ln_parser_t *parsers; /* array of parsers to try */ prsid_t nparsers; /**< current table size (prsid_t slightly abused) */ struct { unsigned isTerminal:1; /**< designates this node a terminal sequence */ unsigned visited:1; /**< work var for recursive procedures */ } flags; struct json_object *tags; /**< tags to assign to events of this type */ int refcnt; /**< reference count for deleting tracking */ struct { unsigned called; unsigned backtracked; /**< incremented when backtracking was initiated */ unsigned terminated; } stats; /**< usage statistics */ const char *rb_id; /**< human-readable rulebase identifier, for stats etc */ // experimental, move outside later const char *rb_file; unsigned int rb_lineno; }; #ifdef ADVANCED_STATS struct advstats { int pathlen; int parser_calls; /**< parser calls in general during path */ int lit_parser_calls; /**< same just for the literal parser */ int backtracked; int recursion_level; es_str_t *exec_path; }; #define ADVSTATS_MAX_ENTITIES 100 extern int advstats_max_pathlen; extern int advstats_pathlens[ADVSTATS_MAX_ENTITIES]; extern int advstats_max_backtracked; extern int advstats_backtracks[ADVSTATS_MAX_ENTITIES]; #endif /** the "normalization parameter block" (npb) * This structure is passed to all normalization routines including * parsers. It contains data that commonly needs to be passed, * like the to be parsed string and its length, as well as read/write * data which is used to track information over the general * normalization process (like the execution path, if requested). * The main purpose is to save stack writes by eliminating the * need for using multiple function parameters. Note that it * must be carefully considered which items to add to the * npb - those that change from recursion level to recursion * level are NOT to be placed here. */ struct npb { ln_ctx ctx; const char *str; /**< to-be-normalized message */ size_t strLen; /**< length of it */ size_t parsedTo; /**< up to which byte could this be parsed? */ es_str_t *rule; /**< a mock-up of the rule used to parse */ es_str_t *exec_path; #ifdef ADVANCED_STATS int pathlen; int backtracked; int recursion_level; struct advstats astats; #endif }; /* Methods */ /** * Allocates and initializes a new parse DAG node. * @memberof ln_pdag * * @param[in] ctx current library context. This MUST match the * context of the parent. * @param[in] parent pointer to the new node inside the parent * * @return pointer to new node or NULL on error */ struct ln_pdag* ln_newPDAG(ln_ctx ctx); /** * Free a parse DAG and destruct all members. * @memberof ln_pdag * * @param[in] DAG pointer to pdag to free */ void ln_pdagDelete(struct ln_pdag *DAG); /** * Add parser to dag node. * Works on unoptimized dag. * * @param[in] pdag pointer to pdag to modify * @param[in] parser parser definition * @returns 0 on success, something else otherwise */ int ln_pdagAddParser(ln_ctx ctx, struct ln_pdag **pdag, json_object *); /** * Display the content of a pdag (debug function). * This is a debug aid that spits out a textual representation * of the provided pdag via multiple calls of the debug callback. * * @param DAG pdag to display */ void ln_displayPDAG(ln_ctx ctx); /** * Generate a DOT graph. * Well, actually it does not generate the graph itself, but a * control file that is suitable for the GNU DOT tool. Such a file * can be very useful to understand complex sample databases * (not to mention that it is probably fun for those creating * samples). * The dot commands are appended to the provided string. * * @param[in] DAG pdag to display * @param[out] str string which receives the DOT commands. */ void ln_genDotPDAGGraph(struct ln_pdag *DAG, es_str_t **str); /** * Build a pdag based on the provided string, but only if necessary. * The passed-in DAG is searched and traversed for str. If a node exactly * matching str is found, that node is returned. If no exact match is found, * a new node is added. Existing nodes may be split, if a so-far common * prefix needs to be split in order to add the new node. * * @param[in] DAG root of the current DAG * @param[in] str string to be added * @param[in] offs offset into str where match needs to start * (this is required for recursive calls to handle * common prefixes) * @return NULL on error, otherwise the pdag leaf that * corresponds to the parameters passed. */ struct ln_pdag * ln_buildPDAG(struct ln_pdag *DAG, es_str_t *str, size_t offs); prsid_t ln_parserName2ID(const char *const __restrict__ name); int ln_pdagOptimize(ln_ctx ctx); void ln_fullPdagStats(ln_ctx ctx, FILE *const fp, const int); ln_parser_t * ln_newLiteralParser(ln_ctx ctx, char lit); ln_parser_t* ln_newParser(ln_ctx ctx, json_object *const prscnf); struct ln_type_pdag * ln_pdagFindType(ln_ctx ctx, const char *const __restrict__ name, const int bAdd); void ln_fullPDagStatsDOT(ln_ctx ctx, FILE *const fp); /* friends */ int ln_normalizeRec(npb_t *const __restrict__ npb, struct ln_pdag *dag, const size_t offs, const int bPartialMatch, struct json_object *json, struct ln_pdag **endNode ); #endif /* #ifndef LOGNORM_PDAG_H_INCLUDED */ liblognorm-2.0.8/src/samp.c000066400000000000000000000742131511425433100155660ustar00rootroot00000000000000/* samp.c -- code for ln_samp objects. * This code handles rulebase processing. Rulebases have been called * "sample bases" in the early days of liblognorm, thus the name. * * Copyright 2010-2018 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include #include #include #include #include #include "liblognorm.h" #include "lognorm.h" #include "samp.h" #include "internal.h" #include "parser.h" #include "pdag.h" #include "v1_liblognorm.h" #include "v1_ptree.h" void ln_sampFree(ln_ctx __attribute__((unused)) ctx, struct ln_samp *samp) { free(samp); } static int ln_parseLegacyFieldDescr(ln_ctx ctx, const char *const buf, const size_t lenBuf, size_t *bufOffs, es_str_t **str, json_object **prscnf) { int r = 0; char *cstr; /* for debug mode strings */ char *ftype = NULL; char name[MAX_FIELDNAME_LEN]; size_t iDst; struct json_object *json = NULL; char *ed = NULL; es_size_t i = *bufOffs; es_str_t *edata = NULL; for( iDst = 0 ; iDst < (MAX_FIELDNAME_LEN - 1) && i < lenBuf && buf[i] != ':' ; ++iDst) { name[iDst] = buf[i++]; } name[iDst] = '\0'; if(iDst == (MAX_FIELDNAME_LEN - 1)) { ln_errprintf(ctx, 0, "field name too long in: %s", buf+(*bufOffs)); FAIL(LN_INVLDFDESCR); } if(i == lenBuf) { ln_errprintf(ctx, 0, "field definition wrong in: %s", buf+(*bufOffs)); FAIL(LN_INVLDFDESCR); } if(iDst == 0) { FAIL(LN_INVLDFDESCR); } if(ctx->debug) { ln_dbgprintf(ctx, "parsed field: '%s'", name); } if(buf[i] != ':') { ln_errprintf(ctx, 0, "missing colon in: %s", buf+(*bufOffs)); FAIL(LN_INVLDFDESCR); } ++i; /* skip ':' */ /* parse and process type (trailing whitespace must be trimmed) */ es_emptyStr(*str); size_t j = i; /* scan for terminator */ while(j < lenBuf && buf[j] != ':' && buf[j] != '{' && buf[j] != '%') ++j; /* now trim trailing space backwards */ size_t next = j; --j; while(j >= i && isspace(buf[j])) --j; /* now copy */ while(i <= j) { CHKR(es_addChar(str, buf[i++])); } /* finally move i to consumed position */ i = next; if(i == lenBuf) { ln_errprintf(ctx, 0, "premature end (missing %%?) in: %s", buf+(*bufOffs)); FAIL(LN_INVLDFDESCR); } ftype = es_str2cstr(*str, NULL); ln_dbgprintf(ctx, "field type '%s', i %d", ftype, i); if(buf[i] == '{') { struct json_tokener *tokener = json_tokener_new(); json = json_tokener_parse_ex(tokener, buf+i, (int) (lenBuf - i)); if(json == NULL) { ln_errprintf(ctx, 0, "invalid json in '%s'", buf+i); } i += tokener->char_offset; json_tokener_free(tokener); } if(buf[i] == '%') { i++; } else { /* parse extra data */ CHKN(edata = es_newStr(8)); i++; while(i < lenBuf) { if(buf[i] == '%') { ++i; break; /* end of field */ } CHKR(es_addChar(&edata, buf[i++])); } es_unescapeStr(edata); if(ctx->debug) { cstr = es_str2cstr(edata, NULL); ln_dbgprintf(ctx, "parsed extra data: '%s'", cstr); free(cstr); } } struct json_object *val; *prscnf = json_object_new_object(); CHKN(val = json_object_new_string(name)); json_object_object_add(*prscnf, "name", val); CHKN(val = json_object_new_string(ftype)); json_object_object_add(*prscnf, "type", val); if(edata != NULL) { ed = es_str2cstr(edata, " "); CHKN(val = json_object_new_string(ed)); json_object_object_add(*prscnf, "extradata", val); } if(json != NULL) { /* now we need to merge the json params into the main object */ struct json_object_iterator it = json_object_iter_begin(json); struct json_object_iterator itEnd = json_object_iter_end(json); while (!json_object_iter_equal(&it, &itEnd)) { struct json_object *const v = json_object_iter_peek_value(&it); json_object_get(v); json_object_object_add(*prscnf, json_object_iter_peek_name(&it), v); json_object_iter_next(&it); } } *bufOffs = i; done: free(ed); if(edata != NULL) es_deleteStr(edata); free(ftype); if(json != NULL) json_object_put(json); return r; } /** * Extract a field description from a sample. * The field description is added to the tail of the current * subtree's field list. The parse buffer must be position on the * leading '%' that starts a field definition. It is a program error * if this condition is not met. * * Note that we break up the object model and access ptree members * directly. Let's consider us a friend of ptree. This is necessary * to optimize the structure for a high-speed parsing process. * * @param[in] str a temporary work string. This is passed in to save the * creation overhead * @returns 0 on success, something else otherwise */ static int addFieldDescr(ln_ctx ctx, struct ln_pdag **pdag, es_str_t *rule, size_t *bufOffs, es_str_t **str) { int r = 0; es_size_t i = *bufOffs; char *ftype = NULL; const char *buf; es_size_t lenBuf; struct json_object *prs_config = NULL; buf = (const char*)es_getBufAddr(rule); lenBuf = es_strlen(rule); assert(buf[i] == '%'); ++i; /* "eat" ':' */ /* skip leading whitespace in field name */ while(i < lenBuf && isspace(buf[i])) ++i; /* check if we have new-style json config */ if(buf[i] == '{' || buf[i] == '[') { struct json_tokener *tokener = json_tokener_new(); prs_config = json_tokener_parse_ex(tokener, buf+i, (int) (lenBuf - i)); i += tokener->char_offset; json_tokener_free(tokener); if(prs_config == NULL || i == lenBuf || buf[i] != '%') { ln_errprintf(ctx, 0, "invalid json in '%s'", buf+i); r = -1; goto done; } *bufOffs = i+1; /* eat '%' - if above ensures it is present */ } else { *bufOffs = i; CHKR(ln_parseLegacyFieldDescr(ctx, buf, lenBuf, bufOffs, str, &prs_config)); } CHKR(ln_pdagAddParser(ctx, pdag, prs_config)); done: free(ftype); return r; } /** * Construct a literal parser json definition. */ static json_object * newLiteralParserJSONConf(char lit) { char buf[] = "x"; buf[0] = lit; struct json_object *val; struct json_object *prscnf = json_object_new_object(); val = json_object_new_string("literal"); json_object_object_add(prscnf, "type", val); val = json_object_new_string(buf); json_object_object_add(prscnf, "text", val); return prscnf; } /** * Parse a Literal string out of the template and add it to the tree. * This function is used to create the unoptimized tree. So we do * one node for each character. These will be compacted by the optimizer * in a later stage. The advantage is that we do not need to care about * splitting the tree. As such the processing is fairly simple: * * for each character in literal (left-to-right): * create literal parser object o * add new DAG node o, advance to it * * @param[in] ctx the context * @param[in/out] subtree on entry, current subtree, on exist newest * deepest subtree * @param[in] rule string with current rule * @param[in/out] bufOffs parse pointer, up to which offset is parsed * (is updated so that it points to first char after consumed * string on exit). * @param str a work buffer, provided to prevent creation of a new object * @return 0 on success, something else otherwise */ static int parseLiteral(ln_ctx ctx, struct ln_pdag **pdag, es_str_t *rule, size_t *const __restrict__ bufOffs, es_str_t **str) { int r = 0; size_t i = *bufOffs; unsigned char *buf = es_getBufAddr(rule); const size_t lenBuf = es_strlen(rule); const char *cstr = NULL; es_emptyStr(*str); while(i < lenBuf) { if(buf[i] == '%') { if(i+1 < lenBuf && buf[i+1] != '%') { break; /* field start is end of literal */ } if (++i == lenBuf) break; } CHKR(es_addChar(str, buf[i])); ++i; } es_unescapeStr(*str); cstr = es_str2cstr(*str, NULL); if(ctx->debug) { ln_dbgprintf(ctx, "parsed literal: '%s'", cstr); } *bufOffs = i; /* we now add the string to the tree */ for(i = 0 ; cstr[i] != '\0' ; ++i) { struct json_object *const prscnf = newLiteralParserJSONConf(cstr[i]); CHKN(prscnf); CHKR(ln_pdagAddParser(ctx, pdag, prscnf)); } r = 0; done: free((void*)cstr); return r; } /* Implementation note: * We read in the sample, and split it into chunks of literal text and * fields. Each literal text is added as whole to the tree, as is each * field individually. To do so, we keep track of our current subtree * root, which changes whenever a new part of the tree is build. It is * set to the then-lowest part of the tree, where the next step sample * data is to be added. * * This function processes the whole string or returns an error. * * format: literal1%field:type:extra-data%literal2 * * @returns the new dag root (or NULL in case of error) */ static int addSampToTree(ln_ctx ctx, es_str_t *rule, ln_pdag *dag, struct json_object *tagBucket) { int r = -1; es_str_t *str = NULL; size_t i; CHKN(str = es_newStr(256)); i = 0; while(i < es_strlen(rule)) { LN_DBGPRINTF(ctx, "addSampToTree %zu of %d", i, es_strlen(rule)); CHKR(parseLiteral(ctx, &dag, rule, &i, &str)); /* After the literal there can be field only*/ if (i < es_strlen(rule)) { CHKR(addFieldDescr(ctx, &dag, rule, &i, &str)); if (i == es_strlen(rule)) { /* finish the tree with empty literal to avoid false merging*/ CHKR(parseLiteral(ctx, &dag, rule, &i, &str)); } } } LN_DBGPRINTF(ctx, "end addSampToTree %zu of %d", i, es_strlen(rule)); /* we are at the end of rule processing, so this node is a terminal */ dag->flags.isTerminal = 1; dag->tags = tagBucket; dag->rb_file = strdup(ctx->conf_file); dag->rb_lineno = ctx->conf_ln_nbr; done: if(str != NULL) es_deleteStr(str); return r; } /** * get the initial word of a rule line that tells us the type of the * line. * @param[in] buf line buffer * @param[in] len length of buffer * @param[out] offs offset after "=" * @param[out] str string with "linetype-word" (newly created) * @returns 0 on success, something else otherwise */ static int getLineType(const char *buf, es_size_t lenBuf, size_t *offs, es_str_t **str) { int r = -1; size_t i; *str = es_newStr(16); for(i = 0 ; i < lenBuf && buf[i] != '=' ; ++i) { CHKR(es_addChar(str, buf[i])); } if(i < lenBuf) ++i; /* skip over '=' */ *offs = i; done: return r; } /** * Get a new common prefix from the config file. That is actually everything from * the current offset to the end of line. * * @param[in] buf line buffer * @param[in] len length of buffer * @param[in] offs offset after "=" * @param[in/out] str string to store common offset. If NULL, it is created, * otherwise it is emptied. * @returns 0 on success, something else otherwise */ static int getPrefix(const char *buf, es_size_t lenBuf, es_size_t offs, es_str_t **str) { int r; if(*str == NULL) { CHKN(*str = es_newStr(lenBuf - offs)); } else { es_emptyStr(*str); } r = es_addBuf(str, (char*)buf + offs, lenBuf - offs); done: return r; } /** * Extend the common prefix. This means that the line is concatenated * to the prefix. This is useful if the same rulebase is to be used with * different prefixes (well, not strictly necessary, but probably useful). * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in] offs offset to-be-added text starts * @returns 0 on success, something else otherwise */ static int extendPrefix(ln_ctx ctx, const char *buf, es_size_t lenBuf, es_size_t offs) { return es_addBuf(&ctx->rulePrefix, (char*)buf+offs, lenBuf - offs); } /** * Add a tag to the tag bucket. Helper to processTags. * @param[in] ctx current context * @param[in] tagname string with tag name * @param[out] tagBucket tagbucket to which new tags shall be added * the tagbucket is created if it is NULL * @returns 0 on success, something else otherwise */ static int addTagStrToBucket(ln_ctx ctx, es_str_t *tagname, struct json_object **tagBucket) { int r = -1; char *cstr; struct json_object *tag; if(*tagBucket == NULL) { CHKN(*tagBucket = json_object_new_array()); } cstr = es_str2cstr(tagname, NULL); ln_dbgprintf(ctx, "tag found: '%s'", cstr); CHKN(tag = json_object_new_string(cstr)); json_object_array_add(*tagBucket, tag); free(cstr); r = 0; done: return r; } /** * Extract the tags and create a tag bucket out of them * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in,out] poffs offset where tags start, on exit and success * offset after tag part (excluding ':') * @param[out] tagBucket tagbucket to which new tags shall be added * the tagbucket is created if it is NULL * @returns 0 on success, something else otherwise */ static int processTags(ln_ctx ctx, const char *buf, es_size_t lenBuf, es_size_t *poffs, struct json_object **tagBucket) { int r = -1; es_str_t *str = NULL; es_size_t i; assert(poffs != NULL); i = *poffs; while(i < lenBuf && buf[i] != ':') { if(buf[i] == ',') { /* end of this tag */ CHKR(addTagStrToBucket(ctx, str, tagBucket)); es_deleteStr(str); str = NULL; } else { if(str == NULL) { CHKN(str = es_newStr(32)); } CHKR(es_addChar(&str, buf[i])); } ++i; } if(buf[i] != ':') goto done; ++i; /* skip ':' */ if(str != NULL) { CHKR(addTagStrToBucket(ctx, str, tagBucket)); es_deleteStr(str); } *poffs = i; r = 0; done: return r; } /** * Process a new rule and add it to pdag. * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in] offs offset where rule starts * @returns 0 on success, something else otherwise */ static int processRule(ln_ctx ctx, const char *buf, es_size_t lenBuf, es_size_t offs) { int r = -1; es_str_t *str; struct json_object *tagBucket = NULL; ln_dbgprintf(ctx, "rule line to add: '%s'", buf+offs); CHKR(processTags(ctx, buf, lenBuf, &offs, &tagBucket)); if(offs == lenBuf) { ln_errprintf(ctx, 0, "error: actual message sample part is missing"); goto done; } if(ctx->rulePrefix == NULL) { CHKN(str = es_newStr(lenBuf)); } else { CHKN(str = es_strdup(ctx->rulePrefix)); } CHKR(es_addBuf(&str, (char*)buf + offs, lenBuf - offs)); addSampToTree(ctx, str, ctx->pdag, tagBucket); es_deleteStr(str); r = 0; done: return r; } static int getTypeName(ln_ctx ctx, const char *const __restrict__ buf, const size_t lenBuf, size_t *const __restrict__ offs, char *const __restrict__ dstbuf) { int r = -1; size_t iDst; size_t i = *offs; if(buf[i] != '@') { ln_errprintf(ctx, 0, "user-defined type name must " "start with '@'"); goto done; } for( iDst = 0 ; i < lenBuf && buf[i] != ':' && iDst < MAX_TYPENAME_LEN - 1 ; ++i, ++iDst) { if(isspace(buf[i])) { ln_errprintf(ctx, 0, "user-defined type name must " "not contain whitespace"); goto done; } dstbuf[iDst] = buf[i]; } dstbuf[iDst] = '\0'; if(i < lenBuf && buf[i] == ':') { r = 0, *offs = i+1; /* skip ":" */ } done: return r; } /** * Process a type definition and add it to the PDAG * disconnected components. * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in] offs offset where rule starts * @returns 0 on success, something else otherwise */ static int processType(ln_ctx ctx, const char *const __restrict__ buf, const size_t lenBuf, size_t offs) { int r = -1; es_str_t *str; char typename[MAX_TYPENAME_LEN]; ln_dbgprintf(ctx, "type line to add: '%s'", buf+offs); CHKR(getTypeName(ctx, buf, lenBuf, &offs, typename)); ln_dbgprintf(ctx, "type name is '%s'", typename); ln_dbgprintf(ctx, "type line to add: '%s'", buf+offs); if(offs == lenBuf) { ln_errprintf(ctx, 0, "error: actual message sample part is missing in type def"); goto done; } // TODO: optimize CHKN(str = es_newStr(lenBuf)); CHKR(es_addBuf(&str, (char*)buf + offs, lenBuf - offs)); struct ln_type_pdag *const td = ln_pdagFindType(ctx, typename, 1); CHKN(td); addSampToTree(ctx, str, td->pdag, NULL); es_deleteStr(str); r = 0; done: return r; } /** * Obtain a field name from a rule base line. * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in/out] offs on entry: offset where tag starts, * on exit: updated offset AFTER TAG and (':') * @param [out] strTag obtained tag, if successful * @returns 0 on success, something else otherwise */ static int getFieldName(ln_ctx __attribute__((unused)) ctx, const char *buf, es_size_t lenBuf, es_size_t *offs, es_str_t **strTag) { int r = -1; es_size_t i; i = *offs; while(i < lenBuf && (isalnum(buf[i]) || buf[i] == '_' || buf[i] == '.')) { if(*strTag == NULL) { CHKN(*strTag = es_newStr(32)); } CHKR(es_addChar(strTag, buf[i])); ++i; } *offs = i; r = 0; done: return r; } /** * Skip over whitespace. * Skips any whitespace present at the offset. * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in/out] offs on entry: offset first unprocessed position */ static void skipWhitespace(ln_ctx __attribute__((unused)) ctx, const char *buf, es_size_t lenBuf, es_size_t *offs) { while(*offs < lenBuf && isspace(buf[*offs])) { (*offs)++; } } /** * Obtain an annotation (field) operation. * This usually is a plus or minus sign followed by a field name * followed (if plus) by an equal sign and the field value. On entry, * offs must be positioned on the first unprocessed field (after ':' for * the initial field!). Extra whitespace is detected and, if present, * skipped. The obtained operation is added to the annotation set provided. * Note that extracted string objects are passed to the annotation; thus it * is vital NOT to free them (most importantly, this is *not* a memory leak). * * @param[in] ctx current context * @param[in] annot active annotation set to which the operation is to be added * @param[in] buf line buffer * @param[in] len length of buffer * @param[in/out] offs on entry: offset where tag starts, * on exit: updated offset AFTER TAG and (':') * @param [out] strTag obtained tag, if successful * @returns 0 on success, something else otherwise */ static int getAnnotationOp(ln_ctx ctx, ln_annot *annot, const char *buf, es_size_t lenBuf, es_size_t *offs) { int r = -1; es_size_t i; es_str_t *fieldName = NULL; es_str_t *fieldVal = NULL; ln_annot_opcode opc; i = *offs; skipWhitespace(ctx, buf, lenBuf, &i); if(i == lenBuf) { r = 0; goto done; /* nothing left to process (no error!) */ } switch(buf[i]) { case '+': opc = ln_annot_ADD; break; case '#': ln_dbgprintf(ctx, "inline comment in 'annotate' line: %s", buf); *offs = lenBuf; r = 0; goto done; case '-': ln_dbgprintf(ctx, "annotate op '-' not yet implemented - failing"); /*FALLTHROUGH*/ default:ln_errprintf(ctx, 0, "invalid annotate operation '%c': %s", buf[i], buf+i); goto fail; } i++; if(i == lenBuf) goto fail; /* nothing left to process */ CHKR(getFieldName(ctx, buf, lenBuf, &i, &fieldName)); if(i == lenBuf) goto fail; /* nothing left to process */ if(buf[i] != '=') goto fail; /* format error */ i++; skipWhitespace(ctx, buf, lenBuf, &i); if(buf[i] != '"') goto fail; /* format error */ ++i; while(i < lenBuf && buf[i] != '"') { if(fieldVal == NULL) { CHKN(fieldVal = es_newStr(32)); } CHKR(es_addChar(&fieldVal, buf[i])); ++i; } *offs = (i == lenBuf) ? i : i+1; CHKR(ln_addAnnotOp(annot, opc, fieldName, fieldVal)); r = 0; done: return r; fail: return -1; } /** * Process a new annotation and add it to the annotation set. * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in] offs offset where annotation starts * @returns 0 on success, something else otherwise */ static int processAnnotate(ln_ctx ctx, const char *buf, es_size_t lenBuf, es_size_t offs) { int r; es_str_t *tag = NULL; ln_annot *annot; ln_dbgprintf(ctx, "sample annotation to add: '%s'", buf+offs); CHKR(getFieldName(ctx, buf, lenBuf, &offs, &tag)); skipWhitespace(ctx, buf, lenBuf, &offs); if(buf[offs] != ':' || tag == NULL) { ln_dbgprintf(ctx, "invalid tag field in annotation, line is '%s'", buf); r=-1; goto done; } ++offs; /* we got an annotation! */ CHKN(annot = ln_newAnnot(tag)); while(offs < lenBuf) { CHKR(getAnnotationOp(ctx, annot, buf, lenBuf, &offs)); } r = ln_addAnnotToSet(ctx->pas, annot); done: return r; } /** * Process include directive. This permits to add unlimited layers * of include files. * * @param[in] ctx current context * @param[in] buf line buffer, a C-string * @param[in] offs offset where annotation starts * @returns 0 on success, something else otherwise */ static int processInclude(ln_ctx ctx, const char *buf, const size_t offs) { int r; const char *const conf_file_save = ctx->conf_file; char *const fname = strdup(buf+offs); size_t lenfname = strlen(fname); const unsigned conf_ln_nbr_save = ctx->conf_ln_nbr; /* trim string - not optimized but also no need to */ for(size_t i = lenfname - 1 ; i > 0 ; --i) { if(isspace(fname[i])) { fname[i] = '\0'; --lenfname; } } CHKR(ln_loadSamples(ctx, fname)); done: free(fname); ctx->conf_file = conf_file_save; ctx->conf_ln_nbr = conf_ln_nbr_save; return r; } /** * Reads a rule (sample) stored in buffer buf and creates a new ln_samp object * out of it, which it adds to the pdag (if required). * * @param[ctx] ctx current library context * @param[buf] cstr buffer containing the string contents of the sample * @param[lenBuf] length of the sample contained within buf * @return standard error code */ static int ln_processSamp(ln_ctx ctx, const char *buf, const size_t lenBuf) { int r = 0; es_str_t *typeStr = NULL; size_t offs; if(getLineType(buf, lenBuf, &offs, &typeStr) != 0) goto done; if(!es_strconstcmp(typeStr, "prefix")) { if(getPrefix(buf, lenBuf, offs, &ctx->rulePrefix) != 0) goto done; } else if(!es_strconstcmp(typeStr, "extendprefix")) { if(extendPrefix(ctx, buf, lenBuf, offs) != 0) goto done; } else if(!es_strconstcmp(typeStr, "rule")) { if(processRule(ctx, buf, lenBuf, offs) != 0) goto done; } else if(!es_strconstcmp(typeStr, "type")) { if(processType(ctx, buf, lenBuf, offs) != 0) goto done; } else if(!es_strconstcmp(typeStr, "annotate")) { if(processAnnotate(ctx, buf, lenBuf, offs) != 0) goto done; } else if(!es_strconstcmp(typeStr, "include")) { CHKR(processInclude(ctx, buf, offs)); } else { char *str; str = es_str2cstr(typeStr, NULL); ln_errprintf(ctx, 0, "invalid record type detected: '%s'", str); free(str); goto done; } done: if(typeStr != NULL) es_deleteStr(typeStr); return r; } /** * Read a character from our sample source. */ static int ln_sampReadChar(const ln_ctx ctx, FILE *const __restrict__ repo, const char **inpbuf) { int c; assert((repo != NULL && inpbuf == NULL) || (repo == NULL && inpbuf != NULL)); if(repo == NULL) { c = (**inpbuf == '\0') ? EOF : *(*inpbuf)++; } else { c = fgetc(repo); } return c; } /* note: comments are only supported at beginning of line! */ /* skip to end of line */ void ln_sampSkipCommentLine(ln_ctx ctx, FILE * const __restrict__ repo, const char **inpbuf) { int c; do { c = ln_sampReadChar(ctx, repo, inpbuf); } while(c != EOF && c != '\n'); ++ctx->conf_ln_nbr; } /* this checks if in a multi-line rule, the next line seems to be a new * rule, which would meand we have some unmatched percent signs inside * our rule (what we call a "runaway rule"). This can easily happen and * is otherwise hard to debug, so let's see if it is the case... * @return 1 if this is a runaway rule, 0 if not */ int ln_sampChkRunawayRule(ln_ctx ctx, FILE *const __restrict__ repo, const char **inpbuf) { int r = 1; fpos_t fpos; char buf[6]; int cont = 1; int read; fgetpos(repo, &fpos); while(cont) { fpos_t inner_fpos; fgetpos(repo, &inner_fpos); if((read = fread(buf, sizeof(char), sizeof(buf)-1, repo)) == 0) { r = 0; goto done; } if(buf[0] == '\n') { fsetpos(repo, &inner_fpos); if(fread(buf, sizeof(char), 1, repo)) {}; /* skip '\n' */ continue; } else if(buf[0] == '#') { fsetpos(repo, &inner_fpos); const unsigned conf_ln_nbr_save = ctx->conf_ln_nbr; ln_sampSkipCommentLine(ctx, repo, inpbuf); ctx->conf_ln_nbr = conf_ln_nbr_save; continue; } if(read != 5) goto done; /* cannot be a rule= line! */ cont = 0; /* no comment, so we can decide */ buf[5] = '\0'; if(!strncmp(buf, "rule=", 5)) { ln_errprintf(ctx, 0, "line has 'rule=' at begin of line, which " "does look like a typo in the previous lines (unmatched " "%% character) and is forbidden. If valid, please re-format " "the rule to start with other characters. Rule ignored."); goto done; } } r = 0; done: fsetpos(repo, &fpos); return r; } /** * Read a rule (sample) from repository (sequentially). * * Reads a sample starting with the current file position and * creates a new ln_samp object out of it, which it adds to the * pdag. * * @param[in] ctx current library context * @param[in] repo repository descriptor if file input is desired * @param[in/out] ptr to ptr of input buffer; this is used if a string is * provided instead of a file. If so, this pointer is advanced * as data is consumed. * @param[out] isEof must be set to 0 on entry and is switched to 1 if EOF occurred. * @return standard error code */ static int ln_sampRead(ln_ctx ctx, FILE *const __restrict__ repo, const char **inpbuf, int *const __restrict__ isEof) { int r = 0; char buf[64*1024]; /**< max size of rule - TODO: make configurable */ size_t i = 0; int inParser = 0; int done = 0; while(!done) { const int c = ln_sampReadChar(ctx, repo, inpbuf); if(c == EOF) { *isEof = 1; if(i == 0) goto done; else done = 1; /* last line missing LF, still process it! */ } else if(c == '\n') { ++ctx->conf_ln_nbr; if(inParser && repo != NULL) { if(ln_sampChkRunawayRule(ctx, repo, inpbuf)) { /* ignore previous rule */ inParser = 0; i = 0; } } if(!inParser && i != 0) done = 1; } else if(c == '#' && i == 0) { ln_sampSkipCommentLine(ctx, repo, inpbuf); i = 0; /* back to beginning */ } else { if(c == '%') inParser = (inParser) ? 0 : 1; buf[i++] = c; if(i >= sizeof(buf)) { ln_errprintf(ctx, 0, "line is too long"); goto done; } } } buf[i] = '\0'; ln_dbgprintf(ctx, "read rulebase line[~%d]: '%s'", ctx->conf_ln_nbr, buf); CHKR(ln_processSamp(ctx, buf, i)); done: return r; } /* check rulebase format version. Returns 2 if this is v2 rulebase, * 1 for any pre-v2 and -1 if there was a problem reading the file. */ static int checkVersion(FILE *const fp) { char buf[64]; if(fgets(buf, sizeof(buf), fp) == NULL) return -1; if(!strcmp(buf, "version=2\n")) { return 2; } else { return 1; } } /* we have a v1 rulebase, so let's do all stuff that we need * to make that ole piece of ... work. */ static int doOldCruft(ln_ctx ctx, const char *file) { int r = -1; if((ctx->ptree = ln_newPTree(ctx, NULL)) == NULL) { free(ctx); r = -1; goto done; } r = ln_v1_loadSamples(ctx, file); done: return r; } /* try to open a rulebase file. This also tries to see if we need to * load it from some pre-configured alternative location. * @returns open file pointer or NULL in case of error */ static FILE * tryOpenRBFile(ln_ctx ctx, const char *const file) { FILE *repo = NULL; if((repo = fopen(file, "r")) != NULL) goto done; const int eno1 = errno; const char *const rb_lib = getenv("LIBLOGNORM_RULEBASES"); if(rb_lib == NULL || *file == '/') { ln_errprintf(ctx, eno1, "cannot open rulebase '%s'", file); goto done; } char *fname = NULL; int len; len = asprintf(&fname, (rb_lib[strlen(rb_lib)-1] == '/') ? "%s%s" : "%s/%s", rb_lib, file); if(len == -1) { ln_errprintf(ctx, errno, "alloc error: cannot open rulebase '%s'", file); goto done; } if((repo = fopen(fname, "r")) == NULL) { const int eno2 = errno; ln_errprintf(ctx, eno1, "cannot open rulebase '%s'", file); ln_errprintf(ctx, eno2, "also tried to locate %s via " "rulebase directory without success. Expanded " "name was '%s'", file, fname); } free(fname); done: return repo; } /* @return 0 if all is ok, 1 if an error occurred */ int ln_sampLoad(ln_ctx ctx, const char *file) { int r = 1; FILE *repo; int isEof = 0; ln_dbgprintf(ctx, "loading rulebase file '%s'", file); if(file == NULL) goto done; if((repo = tryOpenRBFile(ctx, file)) == NULL) goto done; const int version = checkVersion(repo); ln_dbgprintf(ctx, "rulebase version is %d\n", version); if(version == -1) { ln_errprintf(ctx, errno, "error determining version of %s", file); goto done; } if(ctx->version != 0 && version != ctx->version) { ln_errprintf(ctx, errno, "rulebase '%s' must be version %d, but is version %d " " - can not be processed", file, ctx->version, version); goto done; } ctx->version = version; if(ctx->version == 1) { fclose(repo); r = doOldCruft(ctx, file); goto done; } /* now we are in our native code */ ++ctx->conf_ln_nbr; /* "version=2" is line 1! */ while(!isEof) { CHKR(ln_sampRead(ctx, repo, NULL, &isEof)); } fclose(repo); r = 0; if(ctx->include_level == 1) ln_pdagOptimize(ctx); done: return r; } /* @return 0 if all is ok, 1 if an error occurred */ int ln_sampLoadFromString(ln_ctx ctx, const char *string) { int r = 1; int isEof = 0; if(string == NULL) goto done; ln_dbgprintf(ctx, "loading v2 rulebase from string '%s'", string); ctx->version = 2; while(!isEof) { CHKR(ln_sampRead(ctx, NULL, &string, &isEof)); } r = 0; if(ctx->include_level == 1) ln_pdagOptimize(ctx); done: return r; } liblognorm-2.0.8/src/samp.h000066400000000000000000000035401511425433100155660ustar00rootroot00000000000000/** * @file samples.h * @brief Object to process log samples. * @author Rainer Gerhards * * This object handles log samples, and in actual log sample files. * It co-operates with the ptree object to build the actual parser tree. *//* * * liblognorm - a fast samples-based log normalization library * Copyright 2010-2015 by Rainer Gerhards and Adiscon GmbH. * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General PublicCH License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef LIBLOGNORM_SAMPLES_H_INCLUDED #define LIBLOGNORM_SAMPLES_H_INCLUDED #include /* we need es_size_t */ #include /** * A single log sample. */ struct ln_samp { es_str_t *msg; }; void ln_sampFree(ln_ctx ctx, struct ln_samp *samp); int ln_sampLoad(ln_ctx ctx, const char *file); int ln_sampLoadFromString(ln_ctx ctx, const char *string); /* dual-use funtions for v1 engine */ void ln_sampSkipCommentLine(ln_ctx ctx, FILE * const __restrict__ repo, const char **inpbuf); int ln_sampChkRunawayRule(ln_ctx ctx, FILE *const __restrict__ repo, const char **inpbuf); #endif /* #ifndef LIBLOGNORM_SAMPLES_H_INCLUDED */ liblognorm-2.0.8/src/v1_liblognorm.c000066400000000000000000000047371511425433100174040ustar00rootroot00000000000000/* This file implements the liblognorm API. * See header file for descriptions. * * liblognorm - a fast samples-based log normalization library * Copyright 2013-2015 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include "liblognorm.h" #include "v1_liblognorm.h" #include "v1_ptree.h" #include "lognorm.h" #include "annot.h" #include "v1_samp.h" #define ERR_ABORT {r = 1; goto done; } #define CHECK_CTX \ if(ctx->objID != LN_ObjID_CTX) { \ r = -1; \ goto done; \ } ln_ctx ln_v1_inherittedCtx(ln_ctx parent) { ln_ctx child = ln_initCtx(); if (child != NULL) { child->opts = parent->opts; child->dbgCB = parent->dbgCB; child->dbgCookie = parent->dbgCookie; child->version = parent->version; child->ptree = ln_newPTree(child, NULL); } return child; } int ln_v1_loadSample(ln_ctx ctx, const char *buf) { // Something bad happened - no new sample if (ln_v1_processSamp(ctx, buf, strlen(buf)) == NULL) { return 1; } return 0; } int ln_v1_loadSamples(ln_ctx ctx, const char *file) { int r = 0; FILE *repo; struct ln_v1_samp *samp; int isEof = 0; char *fn_to_free = NULL; CHECK_CTX; ctx->conf_file = fn_to_free = strdup(file); ctx->conf_ln_nbr = 0; if(file == NULL) ERR_ABORT; if((repo = fopen(file, "r")) == NULL) { ln_errprintf(ctx, errno, "cannot open file %s", file); ERR_ABORT; } while(!isEof) { if((samp = ln_v1_sampRead(ctx, repo, &isEof)) == NULL) { /* TODO: what exactly to do? */ } } fclose(repo); ctx->conf_file = NULL; done: free((void*)fn_to_free); return r; } liblognorm-2.0.8/src/v1_liblognorm.h000066400000000000000000000131041511425433100173750ustar00rootroot00000000000000/** * @file liblognorm.h * @brief The public liblognorm API. * * Functions other than those defined here MUST not be called by * a liblognorm "user" application. * * This file is meant to be included by applications using liblognorm. * For lognorm library files themselves, include "lognorm.h". *//** * @mainpage * Liblognorm is an easy to use and fast samples-based log normalization * library. * * It can be passed a stream of arbitrary log messages, one at a time, and for * each message it will output well-defined name-value pairs and a set of * tags describing the message. * * For further details, see it's initial announcement available at * https://rainer.gerhards.net/2010/10/introducing-liblognorm.html * * The public interface of this library is describe in liblognorm.h. * * Liblognorm fully supports Unicode. Like most Linux tools, it operates * on UTF-8 natively, called "passive mode". This was decided because we * so can keep the size of data structures small while still supporting * all of the world's languages (actually more than when we did UCS-2). * * At the technical level, we can handle UTF-8 multibyte sequences transparently. * Liblognorm needs to look at a few US-ASCII characters to do the * sample base parsing (things to indicate fields), so this is no * issue. Inside the parse tree, a multibyte sequence can simple be processed * as if it were a sequence of different characters that make up a their * own symbol each. In fact, this even allows for somewhat greater parsing * speed. *//* * * liblognorm - a fast samples-based log normalization library * Copyright 2010-2013 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef V1_LIBLOGNORM_H_INCLUDED #define V1_LIBLOGNORM_H_INCLUDED #include "liblognorm.h" /** * Inherit control attributes from a library context. * * This does not copy the parse-tree, but does copy * behaviour-controlling attributes such as enableRegex. * * Just as with ln_initCtx, ln_exitCtx() must be called on a library * context that is no longer needed. * * @return new library context or NULL if an error occurred */ ln_ctx ln_v1_inherittedCtx(ln_ctx parent); /** * Reads a sample stored in buffer buf and creates a new ln_samp object * out of it. * * @note * It is the caller's responsibility to delete the newly * created ln_samp object if it is no longer needed. * * @param[ctx] ctx current library context * @param[buf] NULL terminated cstr containing the contents of the sample * @return Returns zero on success, something else otherwise. */ int ln_v1_loadSample(ln_ctx ctx, const char *buf); /** * Load a (log) sample file. * * The file must contain log samples in syntactically correct format. Samples are added * to set already loaded in the current context. If there is a sample with duplicate * semantics, this sample will be ignored. Most importantly, this can \b not be used * to change tag assignments for a given sample. * * @param[in] ctx The library context to apply callback to. * @param[in] file Name of file to be loaded. * * @return Returns zero on success, something else otherwise. */ int ln_v1_loadSamples(ln_ctx ctx, const char *file); /** * Normalize a message. * * This is the main library entry point. It is called with a message * to normalize and will return a normalized in-memory representation * of it. * * If an error occurs, the function returns -1. In that case, an * in-memory event representation has been generated if event is * non-NULL. In that case, the event contains further error details in * normalized form. * * @note * This function works on byte-counted strings and as such is able to * process NUL bytes if they occur inside the message. On the other hand, * this means the the correct messages size, \b excluding the NUL byte, * must be provided. * * @param[in] ctx The library context to use. * @param[in] str The message string (see note above). * @param[in] strLen The length of the message in bytes. * @param[out] json_p A new event record or NULL if an error occurred. Must be * destructed if no longer needed. * * @return Returns zero on success, something else otherwise. */ int ln_v1_normalize(ln_ctx ctx, const char *str, size_t strLen, struct json_object **json_p); /** * create a single sample. */ struct ln_v1_samp* ln_v1_sampCreate(ln_ctx __attribute__((unused)) ctx); /* here we add some stuff from the compatibility layer. A separate include * would be cleaner, but would potentially require changes all over the * place. So doing it here is better. The respective replacement * functions should usually be found under ./compat -- rgerhards, 2015-05-20 */ #endif /* #ifndef LOGNORM_H_INCLUDED */ liblognorm-2.0.8/src/v1_parser.c000066400000000000000000002447661511425433100165440ustar00rootroot00000000000000/* * liblognorm - a fast samples-based log normalization library * Copyright 2010-2018 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include #include #include #include #include #include #include "v1_liblognorm.h" #include "internal.h" #include "lognorm.h" #include "v1_parser.h" #include "v1_samp.h" #ifdef FEATURE_REGEXP #include #include #endif /* some helpers */ static inline int hParseInt(const unsigned char **buf, size_t *lenBuf) { const unsigned char *p = *buf; size_t len = *lenBuf; int i = 0; while(len > 0 && isdigit(*p)) { i = i * 10 + *p - '0'; ++p; --len; } *buf = p; *lenBuf = len; return i; } /* parsers for the primitive types * * All parsers receive * * @param[in] str the to-be-parsed string * @param[in] strLen length of the to-be-parsed string * @param[in] offs an offset into the string * @param[in] node fieldlist with additional data; for simple * parsers, this sets variable "ed", which just is * string data. * @param[out] parsed bytes * @param[out] value ptr to json object containing parsed data * (can be unused, but if used *value MUST be NULL on entry) * * They will try to parse out "their" object from the string. If they * succeed, they: * * return 0 on success and LN_WRONGPARSER if this parser could * not successfully parse (but all went well otherwise) and something * else in case of an error. */ #define PARSER(ParserName) \ int ln_parse##ParserName(const char *const str, const size_t strLen, \ size_t *const offs, \ __attribute__((unused)) const ln_fieldList_t *node, \ size_t *parsed, \ __attribute__((unused)) struct json_object **value) \ { \ int r = LN_WRONGPARSER; \ __attribute__((unused)) es_str_t *ed = node->data; \ *parsed = 0; #define FAILParser \ goto parserdone; /* suppress warnings */ \ parserdone: \ r = 0; \ goto done; /* suppress warnings */ \ done: #define ENDFailParser \ return r; \ } /** * Utilities to allow constructors of complex parser's to * easily process field-declaration arguments. */ #define FIELD_ARG_SEPERATOR ":" #define MAX_FIELD_ARGS 10 struct pcons_args_s { int argc; char *argv[MAX_FIELD_ARGS]; }; typedef struct pcons_args_s pcons_args_t; static void free_pcons_args(pcons_args_t** dat_p) { pcons_args_t *dat = *dat_p; *dat_p = NULL; if (! dat) { return; } while((--(dat->argc)) >= 0) { if (dat->argv[dat->argc] != NULL) free(dat->argv[dat->argc]); } free(dat); } static pcons_args_t* pcons_args(es_str_t *args, int expected_argc) { pcons_args_t *dat = NULL; char* orig_str = NULL; if ((dat = malloc(sizeof(pcons_args_t))) == NULL) goto fail; dat->argc = 0; if (args != NULL) { orig_str = es_str2cstr(args, NULL); char *str = orig_str; while (dat->argc < MAX_FIELD_ARGS) { int i = dat->argc++; char *next = (dat->argc == expected_argc) ? NULL : strstr(str, FIELD_ARG_SEPERATOR); if (next == NULL) { if ((dat->argv[i] = strdup(str)) == NULL) goto fail; break; } else { if ((dat->argv[i] = strndup(str, next - str)) == NULL) goto fail; next++; } str = next; } } goto done; fail: if (dat != NULL) free_pcons_args(&dat); done: if (orig_str != NULL) free(orig_str); return dat; } static const char* pcons_arg(pcons_args_t *dat, int i, const char* dflt_val) { if (i >= dat->argc) return dflt_val; return dat->argv[i]; } static char* pcons_arg_copy(pcons_args_t *dat, int i, const char* dflt_val) { const char *str = pcons_arg(dat, i, dflt_val); return (str == NULL) ? NULL : strdup(str); } static void pcons_unescape_arg(pcons_args_t *dat, int i) { char *arg = (char*) pcons_arg(dat, i, NULL); es_str_t *str = NULL; if (arg != NULL) { str = es_newStrFromCStr(arg, strlen(arg)); if (str != NULL) { es_unescapeStr(str); free(arg); dat->argv[i] = es_str2cstr(str, NULL); es_deleteStr(str); } } } /** * Parse a TIMESTAMP as specified in RFC5424 (subset of RFC3339). */ PARSER(RFC5424Date) const unsigned char *pszTS; /* variables to temporarily hold time information while we parse */ __attribute__((unused)) int year; int month; int day; int hour; /* 24 hour clock */ int minute; int second; __attribute__((unused)) int secfrac; /* fractional seconds (must be 32 bit!) */ __attribute__((unused)) int secfracPrecision; int OffsetHour; /* UTC offset in hours */ int OffsetMinute; /* UTC offset in minutes */ size_t len; size_t orglen; /* end variables to temporarily hold time information while we parse */ pszTS = (unsigned char*) str + *offs; len = orglen = strLen - *offs; year = hParseInt(&pszTS, &len); /* We take the liberty to accept slightly malformed timestamps e.g. in * the format of 2003-9-1T1:0:0. */ if(len == 0 || *pszTS++ != '-') goto done; --len; month = hParseInt(&pszTS, &len); if(month < 1 || month > 12) goto done; if(len == 0 || *pszTS++ != '-') goto done; --len; day = hParseInt(&pszTS, &len); if(day < 1 || day > 31) goto done; if(len == 0 || *pszTS++ != 'T') goto done; --len; hour = hParseInt(&pszTS, &len); if(hour < 0 || hour > 23) goto done; if(len == 0 || *pszTS++ != ':') goto done; --len; minute = hParseInt(&pszTS, &len); if(minute < 0 || minute > 59) goto done; if(len == 0 || *pszTS++ != ':') goto done; --len; second = hParseInt(&pszTS, &len); if(second < 0 || second > 60) goto done; /* Now let's see if we have secfrac */ if(len > 0 && *pszTS == '.') { --len; const unsigned char *pszStart = ++pszTS; secfrac = hParseInt(&pszTS, &len); secfracPrecision = (int) (pszTS - pszStart); } else { secfracPrecision = 0; secfrac = 0; } /* check the timezone */ if(len == 0) goto done; if(*pszTS == 'Z') { --len; pszTS++; /* eat Z */ } else if((*pszTS == '+') || (*pszTS == '-')) { --len; pszTS++; OffsetHour = hParseInt(&pszTS, &len); if(OffsetHour < 0 || OffsetHour > 23) goto done; if(len == 0 || *pszTS++ != ':') goto done; --len; OffsetMinute = hParseInt(&pszTS, &len); if(OffsetMinute < 0 || OffsetMinute > 59) goto done; } else { /* there MUST be TZ information */ goto done; } if(len > 0) { if(*pszTS != ' ') /* if it is not a space, it can not be a "good" time */ goto done; } /* we had success, so update parse pointer */ *parsed = orglen - len; r = 0; /* success */ done: return r; } /** * Parse a RFC3164 Date. */ PARSER(RFC3164Date) const unsigned char *p; size_t len, orglen; /* variables to temporarily hold time information while we parse */ __attribute__((unused)) int month; int day; #if 0 /* TODO: why does this still exist? */ int year = 0; /* 0 means no year provided */ #endif int hour; /* 24 hour clock */ int minute; int second; p = (unsigned char*) str + *offs; orglen = len = strLen - *offs; /* If we look at the month (Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec), * we may see the following character sequences occur: * * J(an/u(n/l)), Feb, Ma(r/y), A(pr/ug), Sep, Oct, Nov, Dec * * We will use this for parsing, as it probably is the * fastest way to parse it. */ if(len < 3) goto done; switch(*p++) { case 'j': case 'J': if(*p == 'a' || *p == 'A') { ++p; if(*p == 'n' || *p == 'N') { ++p; month = 1; } else goto done; } else if(*p == 'u' || *p == 'U') { ++p; if(*p == 'n' || *p == 'N') { ++p; month = 6; } else if(*p == 'l' || *p == 'L') { ++p; month = 7; } else goto done; } else goto done; break; case 'f': case 'F': if(*p == 'e' || *p == 'E') { ++p; if(*p == 'b' || *p == 'B') { ++p; month = 2; } else goto done; } else goto done; break; case 'm': case 'M': if(*p == 'a' || *p == 'A') { ++p; if(*p == 'r' || *p == 'R') { ++p; month = 3; } else if(*p == 'y' || *p == 'Y') { ++p; month = 5; } else goto done; } else goto done; break; case 'a': case 'A': if(*p == 'p' || *p == 'P') { ++p; if(*p == 'r' || *p == 'R') { ++p; month = 4; } else goto done; } else if(*p == 'u' || *p == 'U') { ++p; if(*p == 'g' || *p == 'G') { ++p; month = 8; } else goto done; } else goto done; break; case 's': case 'S': if(*p == 'e' || *p == 'E') { ++p; if(*p == 'p' || *p == 'P') { ++p; month = 9; } else goto done; } else goto done; break; case 'o': case 'O': if(*p == 'c' || *p == 'C') { ++p; if(*p == 't' || *p == 'T') { ++p; month = 10; } else goto done; } else goto done; break; case 'n': case 'N': if(*p == 'o' || *p == 'O') { ++p; if(*p == 'v' || *p == 'V') { ++p; month = 11; } else goto done; } else goto done; break; case 'd': case 'D': if(*p == 'e' || *p == 'E') { ++p; if(*p == 'c' || *p == 'C') { ++p; month = 12; } else goto done; } else goto done; break; default: goto done; } len -= 3; /* done month */ if(len == 0 || *p++ != ' ') goto done; --len; /* we accept a slightly malformed timestamp with one-digit days. */ if(*p == ' ') { --len; ++p; } day = hParseInt(&p, &len); if(day < 1 || day > 31) goto done; if(len == 0 || *p++ != ' ') goto done; --len; /* time part */ hour = hParseInt(&p, &len); if(hour > 1970 && hour < 2100) { /* if so, we assume this actually is a year. This is a format found * e.g. in Cisco devices. * year = hour; */ /* re-query the hour, this time it must be valid */ if(len == 0 || *p++ != ' ') goto done; --len; hour = hParseInt(&p, &len); } if(hour < 0 || hour > 23) goto done; if(len == 0 || *p++ != ':') goto done; --len; minute = hParseInt(&p, &len); if(minute < 0 || minute > 59) goto done; if(len == 0 || *p++ != ':') goto done; --len; second = hParseInt(&p, &len); if(second < 0 || second > 60) goto done; /* we provide support for an extra ":" after the date. While this is an * invalid format, it occurs frequently enough (e.g. with Cisco devices) * to permit it as a valid case. -- rgerhards, 2008-09-12 */ if(len > 0 && *p == ':') { ++p; /* just skip past it */ --len; } /* we had success, so update parse pointer */ *parsed = orglen - len; r = 0; /* success */ done: return r; } /** * Parse a Number. * Note that a number is an abstracted concept. We always represent it * as 64 bits (but may later change our mind if performance dictates so). */ PARSER(Number) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; for (i = *offs; i < strLen && isdigit(c[i]); i++); if (i == *offs) goto done; /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /** * Parse a Real-number in floating-pt form. */ PARSER(Float) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; int seen_point = 0; i = *offs; if (c[i] == '-') i++; for (; i < strLen; i++) { if (c[i] == '.') { if (seen_point != 0) break; seen_point = 1; } else if (! isdigit(c[i])) { break; } } if (i == *offs) goto done; /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /** * Parse a hex Number. * A hex number begins with 0x and contains only hex digits until the terminating * whitespace. Note that if a non-hex character is detected inside the number string, * this is NOT considered to be a number. */ PARSER(HexNumber) const char *c; size_t i = *offs; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; if(c[i] != '0' || c[i+1] != 'x') goto done; for (i += 2 ; i < strLen && isxdigit(c[i]); i++); if (i == *offs || !isspace(c[i])) goto done; /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /** * Parse a kernel timestamp. * This is a fixed format, see * https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/kernel/printk/printk.c?id=refs/tags/v4.0#n1011 * This is the code that generates it: * sprintf(buf, "[%5lu.%06lu] ", (unsigned long)ts, rem_nsec / 1000); * We accept up to 12 digits for ts, everything above that for sure is * no timestamp. */ #define LEN_KERNEL_TIMESTAMP 14 PARSER(KernelTimestamp) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; if(c[i] != '[' || i+LEN_KERNEL_TIMESTAMP > strLen || !isdigit(c[i+1]) || !isdigit(c[i+2]) || !isdigit(c[i+3]) || !isdigit(c[i+4]) || !isdigit(c[i+5]) ) goto done; i += 6; for(int j = 0 ; j < 7 && i < strLen && isdigit(c[i]) ; ) ++i, ++j; /* just scan */ if(i >= strLen || c[i] != '.') goto done; ++i; /* skip over '.' */ if( i+7 > strLen || !isdigit(c[i+0]) || !isdigit(c[i+1]) || !isdigit(c[i+2]) || !isdigit(c[i+3]) || !isdigit(c[i+4]) || !isdigit(c[i+5]) || c[i+6] != ']' ) goto done; i += 7; /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /** * Parse whitespace. * This parses all whitespace until the first non-whitespace character * is found. This is primarily a tool to skip to the next "word" if * the exact number of whitespace characters (and type of whitespace) * is not known. The current parsing position MUST be on a whitespace, * else the parser does not match. * This parser is also a forward-compatibility tool for the upcoming * slsa (simple log structure analyser) tool. */ PARSER(Whitespace) const char *c; size_t i = *offs; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; if(!isspace(c[i])) goto done; for (i++ ; i < strLen && isspace(c[i]); i++); /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /** * Parse a word. * A word is a SP-delimited entity. The parser always works, except if * the offset is position on a space upon entry. */ PARSER(Word) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; /* search end of word */ while(i < strLen && c[i] != ' ') i++; if(i == *offs) goto done; /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /** * Parse everything up to a specific string. * swisskid, 2015-01-21 */ PARSER(StringTo) const char *c; char *toFind = NULL; size_t i, j, k, m; int chkstr; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); assert(ed != NULL); k = es_strlen(ed) - 1; toFind = es_str2cstr(ed, NULL); c = str; i = *offs; chkstr = 0; /* Total hunt for letter */ while(chkstr == 0 && i < strLen ) { i++; if(c[i] == toFind[0]) { /* Found the first letter, now find the rest of the string */ j = 0; m = i; while(m < strLen && j < k ) { m++; j++; if(c[m] != toFind[j]) break; if (j == k) chkstr = 1; } } } if(i == *offs || i == strLen || c[i] != toFind[0]) goto done; /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: if(toFind != NULL) free(toFind); return r; } /** * Parse a alphabetic word. * A alpha word is composed of characters for which isalpha returns true. * The parser dones if there is no alpha character at all. */ PARSER(Alpha) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; /* search end of word */ while(i < strLen && isalpha(c[i])) i++; if(i == *offs) { goto done; } /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /** * Parse everything up to a specific character. * The character must be the only char inside extra data passed to the parser. * It is a program error if strlen(ed) != 1. It is considered a format error if * a) the to-be-parsed buffer is already positioned on the terminator character * b) there is no terminator until the end of the buffer * In those cases, the parsers declares itself as not being successful, in all * other cases a string is extracted. */ PARSER(CharTo) const char *c; unsigned char cTerm; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); assert(es_strlen(ed) == 1); cTerm = *(es_getBufAddr(ed)); c = str; i = *offs; /* search end of word */ while(i < strLen && c[i] != cTerm) i++; if(i == *offs || i == strLen || c[i] != cTerm) goto done; /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /** * Parse everything up to a specific character, or up to the end of string. * The character must be the only char inside extra data passed to the parser. * It is a program error if strlen(ed) != 1. * This parser always returns success. * By nature of the parser, it is required that end of string or the separator * follows this field in rule. */ PARSER(CharSeparated) const char *c; unsigned char cTerm; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); assert(es_strlen(ed) == 1); cTerm = *(es_getBufAddr(ed)); c = str; i = *offs; /* search end of word */ while(i < strLen && c[i] != cTerm) i++; /* success, persist */ *parsed = i - *offs; r = 0; /* success */ return r; } /** * Parse yet-to-be-matched portion of string by re-applying * top-level rules again. */ #define DEFAULT_REMAINING_FIELD_NAME "tail" struct recursive_parser_data_s { ln_ctx ctx; char* remaining_field; int free_ctx; }; PARSER(Recursive) assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); struct recursive_parser_data_s* pData = (struct recursive_parser_data_s*) node->parser_data; if (pData != NULL) { int remaining_len = strLen - *offs; const char *remaining_str = str + *offs; json_object *unparsed = NULL; CHKN(*value = json_object_new_object()); ln_normalize(pData->ctx, remaining_str, remaining_len, value); if (json_object_object_get_ex(*value, UNPARSED_DATA_KEY, &unparsed)) { json_object_put(*value); *value = NULL; *parsed = 0; } else if (pData->remaining_field != NULL && json_object_object_get_ex(*value, pData->remaining_field, &unparsed)) { *parsed = strLen - *offs - json_object_get_string_len(unparsed); json_object_object_del(*value, pData->remaining_field); } else { *parsed = strLen - *offs; } } r = 0; /* success */ done: return r; } typedef ln_ctx (ctx_constructor)(ln_ctx, pcons_args_t*, const char*); static void* _recursive_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx, int no_of_args, int remaining_field_arg_idx, int free_ctx, ctx_constructor *fn) { int r = LN_BADCONFIG; char* name = NULL; struct recursive_parser_data_s *pData = NULL; pcons_args_t *args = NULL; CHKN(name = es_str2cstr(node->name, NULL)); CHKN(pData = calloc(1, sizeof(struct recursive_parser_data_s))); pData->free_ctx = free_ctx; pData->remaining_field = NULL; CHKN(args = pcons_args(node->raw_data, no_of_args)); CHKN(pData->ctx = fn(ctx, args, name)); CHKN(pData->remaining_field = pcons_arg_copy(args, remaining_field_arg_idx, DEFAULT_REMAINING_FIELD_NAME)); r = 0; done: if (r != 0) { if (name == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for recursive/descent field name"); else if (pData == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for parser-data for field: %s", name); else if (args == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for argument-parsing for field: %s", name); else if (pData->ctx == NULL) ln_dbgprintf(ctx, "recursive/descent normalizer context creation " "doneed for field: %s", name); else if (pData->remaining_field == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for remaining-field name for " "recursive/descent field: %s", name); recursive_parser_data_destructor((void**) &pData); } free(name); free_pcons_args(&args); return pData; } static ln_ctx identity_recursive_parse_ctx_constructor(ln_ctx parent_ctx, __attribute__((unused)) pcons_args_t* args, __attribute__((unused)) const char* field_name) { return parent_ctx; } void* recursive_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx) { return _recursive_parser_data_constructor(node, ctx, 1, 0, 0, identity_recursive_parse_ctx_constructor); } static ln_ctx child_recursive_parse_ctx_constructor(ln_ctx parent_ctx, pcons_args_t* args, const char* field_name) { int r = LN_BADCONFIG; const char* rb = NULL; ln_ctx ctx = NULL; pcons_unescape_arg(args, 0); CHKN(rb = pcons_arg(args, 0, NULL)); CHKN(ctx = ln_v1_inherittedCtx(parent_ctx)); CHKR(ln_v1_loadSamples(ctx, rb)); done: if (r != 0) { if (rb == NULL) ln_dbgprintf(parent_ctx, "file-name for descent rulebase not provided for field: %s", field_name); else if (ctx == NULL) ln_dbgprintf(parent_ctx, "couldn't allocate memory to create descent-field normalizer " "context for field: %s", field_name); else ln_dbgprintf(parent_ctx, "couldn't load samples into descent context for field: %s", field_name); if (ctx != NULL) ln_exitCtx(ctx); ctx = NULL; } return ctx; } void* descent_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx) { return _recursive_parser_data_constructor(node, ctx, 2, 1, 1, child_recursive_parse_ctx_constructor); } void recursive_parser_data_destructor(void** dataPtr) { if (*dataPtr != NULL) { struct recursive_parser_data_s *pData = (struct recursive_parser_data_s*) *dataPtr; if (pData->free_ctx && pData->ctx != NULL) { ln_exitCtx(pData->ctx); pData->ctx = NULL; } if (pData->remaining_field != NULL) free(pData->remaining_field); free(pData); *dataPtr = NULL; } }; /** * Parse string tokenized by given char-sequence * The sequence may appear 0 or more times, but zero times means 1 token. * NOTE: its not 0 tokens, but 1 token. * * The token found is parsed according to the field-type provided after * tokenizer char-seq. */ #define DEFAULT_MATCHED_FIELD_NAME "default" struct tokenized_parser_data_s { es_str_t *tok_str; ln_ctx ctx; char *remaining_field; int use_default_field; int free_ctx; }; typedef struct tokenized_parser_data_s tokenized_parser_data_t; PARSER(Tokenized) assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); tokenized_parser_data_t *pData = (tokenized_parser_data_t*) node->parser_data; if (pData != NULL ) { json_object *json_p = NULL; if (pData->use_default_field) CHKN(json_p = json_object_new_object()); json_object *matches = NULL; CHKN(matches = json_object_new_array()); int remaining_len = strLen - *offs; const char *remaining_str = str + *offs; json_object *remaining = NULL; json_object *match = NULL; while (remaining_len > 0) { if (! pData->use_default_field) { json_object_put(json_p); json_p = json_object_new_object(); } /*TODO: handle null condition gracefully*/ ln_normalize(pData->ctx, remaining_str, remaining_len, &json_p); if (remaining) json_object_put(remaining); if (pData->use_default_field && json_object_object_get_ex(json_p, DEFAULT_MATCHED_FIELD_NAME, &match)) { json_object_array_add(matches, json_object_get(match)); } else if (! (pData->use_default_field || json_object_object_get_ex(json_p, UNPARSED_DATA_KEY, &match))) { json_object_array_add(matches, json_object_get(json_p)); } else { if (json_object_array_length(matches) > 0) { remaining_len += es_strlen(pData->tok_str); break; } else { json_object_put(json_p); json_object_put(matches); FAIL(LN_WRONGPARSER); } } if (json_object_object_get_ex(json_p, pData->remaining_field, &remaining)) { remaining_len = json_object_get_string_len(remaining); if (remaining_len > 0) { remaining_str = json_object_get_string(json_object_get(remaining)); json_object_object_del(json_p, pData->remaining_field); if (es_strbufcmp(pData->tok_str, (const unsigned char *)remaining_str, es_strlen(pData->tok_str))) { json_object_put(remaining); break; } else { remaining_str += es_strlen(pData->tok_str); remaining_len -= es_strlen(pData->tok_str); } } } else { remaining_len = 0; break; } if (pData->use_default_field) json_object_object_del(json_p, DEFAULT_MATCHED_FIELD_NAME); } json_object_put(json_p); /* success, persist */ *parsed = (strLen - *offs) - remaining_len; *value = matches; } else { FAIL(LN_BADPARSERSTATE); } r = 0; /* success */ done: return r; } void tokenized_parser_data_destructor(void** dataPtr) { tokenized_parser_data_t *data = (tokenized_parser_data_t*) *dataPtr; if (data->tok_str != NULL) es_deleteStr(data->tok_str); if (data->free_ctx && (data->ctx != NULL)) ln_exitCtx(data->ctx); if (data->remaining_field != NULL) free(data->remaining_field); free(data); *dataPtr = NULL; } static void load_generated_parser_samples(ln_ctx ctx, const char* const field_descr, const int field_descr_len, const char* const suffix, const int length) { static const char* const RULE_PREFIX = "rule=:%"DEFAULT_MATCHED_FIELD_NAME":";/*TODO: extract nice constants*/ static const int RULE_PREFIX_LEN = 15; char *sample_str = NULL; es_str_t *field_decl = es_newStrFromCStr(RULE_PREFIX, RULE_PREFIX_LEN); if (! field_decl) goto free; if (es_addBuf(&field_decl, field_descr, field_descr_len) || es_addBuf(&field_decl, "%", 1) || es_addBuf(&field_decl, suffix, length)) { ln_dbgprintf(ctx, "couldn't prepare field for tokenized field-picking: '%s'", field_descr); goto free; } sample_str = es_str2cstr(field_decl, NULL); if (! sample_str) { ln_dbgprintf(ctx, "couldn't prepare sample-string for: '%s'", field_descr); goto free; } ln_v1_loadSample(ctx, sample_str); free: if (sample_str) free(sample_str); if (field_decl) es_deleteStr(field_decl); } static ln_ctx generate_context_with_field_as_prefix(ln_ctx parent, const char* field_descr, int field_descr_len) { int r = LN_BADCONFIG; const char* remaining_field = "%"DEFAULT_REMAINING_FIELD_NAME":rest%"; ln_ctx ctx = NULL; CHKN(ctx = ln_v1_inherittedCtx(parent)); load_generated_parser_samples(ctx, field_descr, field_descr_len, remaining_field, strlen(remaining_field)); load_generated_parser_samples(ctx, field_descr, field_descr_len, "", 0); r = 0; done: if (r != 0) { ln_exitCtx(ctx); ctx = NULL; } return ctx; } static ln_fieldList_t* parse_tokenized_content_field(ln_ctx ctx, const char* field_descr, size_t field_descr_len) { es_str_t* tmp = NULL; es_str_t* descr = NULL; ln_fieldList_t *node = NULL; int r = 0; CHKN(tmp = es_newStr(80)); CHKN(descr = es_newStr(80)); const char* field_prefix = "%" DEFAULT_MATCHED_FIELD_NAME ":"; CHKR(es_addBuf(&descr, field_prefix, strlen(field_prefix))); CHKR(es_addBuf(&descr, field_descr, field_descr_len)); CHKR(es_addChar(&descr, '%')); es_size_t offset = 0; CHKN(node = ln_v1_parseFieldDescr(ctx, descr, &offset, &tmp, &r)); if (offset != es_strlen(descr)) FAIL(LN_BADPARSERSTATE); done: if (r != 0) { if (node != NULL) ln_deletePTreeNode(node); node = NULL; } if (descr != NULL) es_deleteStr(descr); if (tmp != NULL) es_deleteStr(tmp); return node; } void* tokenized_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx) { int r = LN_BADCONFIG; char* name = es_str2cstr(node->name, NULL); pcons_args_t *args = NULL; tokenized_parser_data_t *pData = NULL; const char *field_descr = NULL; ln_fieldList_t* field = NULL; const char *tok = NULL; CHKN(args = pcons_args(node->raw_data, 2)); CHKN(pData = calloc(1, sizeof(tokenized_parser_data_t))); pcons_unescape_arg(args, 0); CHKN(tok = pcons_arg(args, 0, NULL)); CHKN(pData->tok_str = es_newStrFromCStr(tok, strlen(tok))); es_unescapeStr(pData->tok_str); CHKN(field_descr = pcons_arg(args, 1, NULL)); const int field_descr_len = strlen(field_descr); pData->free_ctx = 1; CHKN(field = parse_tokenized_content_field(ctx, field_descr, field_descr_len)); if (field->parser == ln_parseRecursive) { pData->use_default_field = 0; struct recursive_parser_data_s *dat = (struct recursive_parser_data_s*) field->parser_data; if (dat != NULL) { CHKN(pData->remaining_field = strdup(dat->remaining_field)); pData->free_ctx = dat->free_ctx; pData->ctx = dat->ctx; dat->free_ctx = 0; } } else { pData->use_default_field = 1; CHKN(pData->ctx = generate_context_with_field_as_prefix(ctx, field_descr, field_descr_len)); } if (pData->remaining_field == NULL) CHKN(pData->remaining_field = strdup(DEFAULT_REMAINING_FIELD_NAME)); r = 0; done: if (r != 0) { if (name == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for tokenized-field name"); else if (args == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for argument-parsing for field: %s", name); else if (pData == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for parser-data for field: %s", name); else if (tok == NULL) ln_dbgprintf(ctx, "token-separator not provided for field: %s", name); else if (pData->tok_str == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for token-separator " "for field: %s", name); else if (field_descr == NULL) ln_dbgprintf(ctx, "field-type not provided for field: %s", name); else if (field == NULL) ln_dbgprintf(ctx, "couldn't resolve single-token field-type for tokenized field: %s", name); else if (pData->ctx == NULL) ln_dbgprintf(ctx, "couldn't initialize normalizer-context for field: %s", name); else if (pData->remaining_field == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for " "remaining-field-name for field: %s", name); if (pData) tokenized_parser_data_destructor((void**) &pData); } if (name != NULL) free(name); if (field != NULL) ln_deletePTreeNode(field); if (args) free_pcons_args(&args); return pData; } #ifdef FEATURE_REGEXP /** * Parse string matched by provided posix extended regex. * * Please note that using regex field in most cases will be * significantly slower than other field-types. */ struct regex_parser_data_s { pcre *re; int consume_group; int return_group; int max_groups; }; PARSER(Regex) assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); unsigned int* ovector = NULL; struct regex_parser_data_s *pData = (struct regex_parser_data_s*) node->parser_data; if (pData != NULL) { ovector = calloc(pData->max_groups, sizeof(unsigned int) * 3); if (ovector == NULL) FAIL(LN_NOMEM); int result = pcre_exec(pData->re, NULL, str, strLen, *offs, 0, (int*) ovector, pData->max_groups * 3); if (result == 0) result = pData->max_groups; if (result > pData->consume_group) { /*please check 'man 3 pcreapi' for cryptic '2 * n' and '2 * n + 1' magic*/ if (ovector[2 * pData->consume_group] == *offs) { *parsed = ovector[2 * pData->consume_group + 1] - ovector[2 * pData->consume_group]; if (pData->consume_group != pData->return_group) { char* val = NULL; if((val = strndup(str + ovector[2 * pData->return_group], ovector[2 * pData->return_group + 1] - ovector[2 * pData->return_group])) == NULL) { free(ovector); FAIL(LN_NOMEM); } *value = json_object_new_string(val); free(val); if (*value == NULL) { free(ovector); FAIL(LN_NOMEM); } } } } free(ovector); } r = 0; /* success */ done: return r; } static const char* regex_parser_configure_consume_and_return_group(pcons_args_t* args, struct regex_parser_data_s *pData) { const char* consume_group_parse_error = "couldn't parse consume-group number"; const char* return_group_parse_error = "couldn't parse return-group number"; char* tmp = NULL; const char* consume_grp_str = NULL; const char* return_grp_str = NULL; if ((consume_grp_str = pcons_arg(args, 1, "0")) == NULL || strlen(consume_grp_str) == 0) return consume_group_parse_error; if ((return_grp_str = pcons_arg(args, 2, consume_grp_str)) == NULL || strlen(return_grp_str) == 0) return return_group_parse_error; errno = 0; pData->consume_group = strtol(consume_grp_str, &tmp, 10); if (errno != 0 || strlen(tmp) != 0) return consume_group_parse_error; pData->return_group = strtol(return_grp_str, &tmp, 10); if (errno != 0 || strlen(tmp) != 0) return return_group_parse_error; return NULL; } void* regex_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx) { int r = LN_BADCONFIG; char* exp = NULL; const char* grp_parse_err = NULL; pcons_args_t* args = NULL; char* name = NULL; struct regex_parser_data_s *pData = NULL; const char *unescaped_exp = NULL; const char *error = NULL; int erroffset = 0; CHKN(name = es_str2cstr(node->name, NULL)); if (! ctx->opts & LN_CTXOPT_ALLOW_REGEX) FAIL(LN_BADCONFIG); CHKN(pData = malloc(sizeof(struct regex_parser_data_s))); pData->re = NULL; CHKN(args = pcons_args(node->raw_data, 3)); pData->consume_group = pData->return_group = 0; CHKN(unescaped_exp = pcons_arg(args, 0, NULL)); pcons_unescape_arg(args, 0); CHKN(exp = pcons_arg_copy(args, 0, NULL)); if ((grp_parse_err = regex_parser_configure_consume_and_return_group(args, pData)) != NULL) FAIL(LN_BADCONFIG); CHKN(pData->re = pcre_compile(exp, 0, &error, &erroffset, NULL)); pData->max_groups = ((pData->consume_group > pData->return_group) ? pData->consume_group : pData->return_group) + 1; r = 0; done: if (r != 0) { if (name == NULL) ln_dbgprintf(ctx, "couldn't allocate memory regex-field name"); else if (! ctx->opts & LN_CTXOPT_ALLOW_REGEX) ln_dbgprintf(ctx, "regex support is not enabled for: '%s' " "(please check lognorm context initialization)", name); else if (pData == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for parser-data for field: %s", name); else if (args == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for argument-parsing for field: %s", name); else if (unescaped_exp == NULL) ln_dbgprintf(ctx, "regular-expression missing for field: '%s'", name); else if (exp == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for regex-string for field: '%s'", name); else if (grp_parse_err != NULL) ln_dbgprintf(ctx, "%s for: '%s'", grp_parse_err, name); else if (pData->re == NULL) ln_dbgprintf(ctx, "couldn't compile regex(encountered error '%s' at char '%d' in pattern) " "for regex-matched field: '%s'", error, erroffset, name); regex_parser_data_destructor((void**)&pData); } if (exp != NULL) free(exp); if (args != NULL) free_pcons_args(&args); if (name != NULL) free(name); return pData; } void regex_parser_data_destructor(void** dataPtr) { if ((*dataPtr) != NULL) { struct regex_parser_data_s *pData = (struct regex_parser_data_s*) *dataPtr; if (pData->re != NULL) pcre_free(pData->re); free(pData); *dataPtr = NULL; } } #endif /** * Parse yet-to-be-matched portion of string by re-applying * top-level rules again. */ typedef enum interpret_type { /* If you change this, be sure to update json_type_to_name() too */ it_b10int, it_b16int, it_floating_pt, it_boolean } interpret_type; struct interpret_parser_data_s { ln_ctx ctx; enum interpret_type intrprt; }; static json_object* interpret_as_int(json_object *value, int base) { if (json_object_is_type(value, json_type_string)) { return json_object_new_int64(strtol(json_object_get_string(value), NULL, base)); } else if (json_object_is_type(value, json_type_int)) { return value; } else { return NULL; } } static json_object* interpret_as_double(json_object *value) { double val = json_object_get_double(value); return json_object_new_double(val); } static json_object* interpret_as_boolean(json_object *value) { json_bool val; if (json_object_is_type(value, json_type_string)) { const char* str = json_object_get_string(value); val = (strcasecmp(str, "false") == 0 || strcasecmp(str, "no") == 0) ? 0 : 1; } else { val = json_object_get_boolean(value); } return json_object_new_boolean(val); } static int reinterpret_value(json_object **value, enum interpret_type to_type) { switch(to_type) { case it_b10int: *value = interpret_as_int(*value, 10); break; case it_b16int: *value = interpret_as_int(*value, 16); break; case it_floating_pt: *value = interpret_as_double(*value); break; case it_boolean: *value = interpret_as_boolean(*value); break; default: return 0; } return 1; } PARSER(Interpret) assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); json_object *unparsed = NULL; json_object *parsed_raw = NULL; struct interpret_parser_data_s* pData = (struct interpret_parser_data_s*) node->parser_data; if (pData != NULL) { int remaining_len = strLen - *offs; const char *remaining_str = str + *offs; CHKN(parsed_raw = json_object_new_object()); ln_normalize(pData->ctx, remaining_str, remaining_len, &parsed_raw); if (json_object_object_get_ex(parsed_raw, UNPARSED_DATA_KEY, NULL)) { *parsed = 0; } else { json_object_object_get_ex(parsed_raw, DEFAULT_MATCHED_FIELD_NAME, value); json_object_object_get_ex(parsed_raw, DEFAULT_REMAINING_FIELD_NAME, &unparsed); if (reinterpret_value(value, pData->intrprt)) { *parsed = strLen - *offs - json_object_get_string_len(unparsed); } } json_object_put(parsed_raw); } r = 0; /* success */ done: return r; } void* interpret_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx) { int r = LN_BADCONFIG; char* name = NULL; struct interpret_parser_data_s *pData = NULL; pcons_args_t *args = NULL; int bad_interpret = 0; const char* type_str = NULL; const char *field_type = NULL; CHKN(name = es_str2cstr(node->name, NULL)); CHKN(pData = calloc(1, sizeof(struct interpret_parser_data_s))); CHKN(args = pcons_args(node->raw_data, 2)); CHKN(type_str = pcons_arg(args, 0, NULL)); if (strcmp(type_str, "int") == 0 || strcmp(type_str, "base10int") == 0) { pData->intrprt = it_b10int; } else if (strcmp(type_str, "base16int") == 0) { pData->intrprt = it_b16int; } else if (strcmp(type_str, "float") == 0) { pData->intrprt = it_floating_pt; } else if (strcmp(type_str, "bool") == 0) { pData->intrprt = it_boolean; } else { bad_interpret = 1; FAIL(LN_BADCONFIG); } CHKN(field_type = pcons_arg(args, 1, NULL)); CHKN(pData->ctx = generate_context_with_field_as_prefix(ctx, field_type, strlen(field_type))); r = 0; done: if (r != 0) { if (name == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for interpret-field name"); else if (pData == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for parser-data for field: %s", name); else if (args == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for argument-parsing for field: %s", name); else if (type_str == NULL) ln_dbgprintf(ctx, "no type provided for interpretation of field: %s", name); else if (bad_interpret != 0) ln_dbgprintf(ctx, "interpretation to unknown type '%s' requested for field: %s", type_str, name); else if (field_type == NULL) ln_dbgprintf(ctx, "field-type to actually match the content not provided for " "field: %s", name); else if (pData->ctx == NULL) ln_dbgprintf(ctx, "couldn't instantiate the normalizer context for matching " "field: %s", name); interpret_parser_data_destructor((void**) &pData); } free(name); free_pcons_args(&args); return pData; } void interpret_parser_data_destructor(void** dataPtr) { if (*dataPtr != NULL) { struct interpret_parser_data_s *pData = (struct interpret_parser_data_s*) *dataPtr; if (pData->ctx != NULL) ln_exitCtx(pData->ctx); free(pData); *dataPtr = NULL; } }; /** * Parse suffixed char-sequence, where suffix is one of many possible suffixes. */ struct suffixed_parser_data_s { int nsuffix; int *suffix_offsets; int *suffix_lengths; char* suffixes_str; ln_ctx ctx; char* value_field_name; char* suffix_field_name; }; PARSER(Suffixed) { assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); json_object *unparsed = NULL; json_object *parsed_raw = NULL; json_object *parsed_value = NULL; json_object *result = NULL; json_object *suffix = NULL; struct suffixed_parser_data_s *pData = (struct suffixed_parser_data_s*) node->parser_data; if (pData != NULL) { int remaining_len = strLen - *offs; const char *remaining_str = str + *offs; int i; CHKN(parsed_raw = json_object_new_object()); ln_normalize(pData->ctx, remaining_str, remaining_len, &parsed_raw); if (json_object_object_get_ex(parsed_raw, UNPARSED_DATA_KEY, NULL)) { *parsed = 0; } else { json_object_object_get_ex(parsed_raw, DEFAULT_MATCHED_FIELD_NAME, &parsed_value); json_object_object_get_ex(parsed_raw, DEFAULT_REMAINING_FIELD_NAME, &unparsed); const char* unparsed_frag = json_object_get_string(unparsed); for(i = 0; i < pData->nsuffix; i++) { const char* possible_suffix = pData->suffixes_str + pData->suffix_offsets[i]; int len = pData->suffix_lengths[i]; if (strncmp(possible_suffix, unparsed_frag, len) == 0) { CHKN(result = json_object_new_object()); CHKN(suffix = json_object_new_string(possible_suffix)); json_object_get(parsed_value); json_object_object_add(result, pData->value_field_name, parsed_value); json_object_object_add(result, pData->suffix_field_name, suffix); *parsed = strLen - *offs - json_object_get_string_len(unparsed) + len; break; } } if (result != NULL) { *value = result; } } } FAILParser if (r != 0) { if (result != NULL) json_object_put(result); } if (parsed_raw != NULL) json_object_put(parsed_raw); } ENDFailParser static struct suffixed_parser_data_s* _suffixed_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx, es_str_t* raw_args, const char* value_field, const char* suffix_field) { int r = LN_BADCONFIG; pcons_args_t* args = NULL; char* name = NULL; struct suffixed_parser_data_s *pData = NULL; const char *escaped_tokenizer = NULL; const char *uncopied_suffixes_str = NULL; const char *tokenizer = NULL; char *suffixes_str = NULL; const char *field_type = NULL; char *tok_saveptr = NULL; char *tok_input = NULL; int i = 0; char *tok = NULL; CHKN(name = es_str2cstr(node->name, NULL)); CHKN(pData = calloc(1, sizeof(struct suffixed_parser_data_s))); if (value_field == NULL) value_field = "value"; if (suffix_field == NULL) suffix_field = "suffix"; pData->value_field_name = strdup(value_field); pData->suffix_field_name = strdup(suffix_field); CHKN(args = pcons_args(raw_args, 3)); CHKN(escaped_tokenizer = pcons_arg(args, 0, NULL)); pcons_unescape_arg(args, 0); CHKN(tokenizer = pcons_arg(args, 0, NULL)); CHKN(uncopied_suffixes_str = pcons_arg(args, 1, NULL)); pcons_unescape_arg(args, 1); CHKN(suffixes_str = pcons_arg_copy(args, 1, NULL)); tok_input = suffixes_str; while (strtok_r(tok_input, tokenizer, &tok_saveptr) != NULL) { tok_input = NULL; pData->nsuffix++; } if (pData->nsuffix == 0) { FAIL(LN_INVLDFDESCR); } CHKN(pData->suffix_offsets = calloc(pData->nsuffix, sizeof(int))); CHKN(pData->suffix_lengths = calloc(pData->nsuffix, sizeof(int))); CHKN(pData->suffixes_str = pcons_arg_copy(args, 1, NULL)); tok_input = pData->suffixes_str; while ((tok = strtok_r(tok_input, tokenizer, &tok_saveptr)) != NULL) { tok_input = NULL; pData->suffix_offsets[i] = tok - pData->suffixes_str; pData->suffix_lengths[i++] = strlen(tok); } CHKN(field_type = pcons_arg(args, 2, NULL)); CHKN(pData->ctx = generate_context_with_field_as_prefix(ctx, field_type, strlen(field_type))); r = 0; done: if (r != 0) { if (name == NULL) ln_dbgprintf(ctx, "couldn't allocate memory suffixed-field name"); else if (pData == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for parser-data for field: %s", name); else if (pData->value_field_name == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for value-field's name for field: %s", name); else if (pData->suffix_field_name == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for suffix-field's name for field: %s", name); else if (args == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for argument-parsing for field: %s", name); else if (escaped_tokenizer == NULL) ln_dbgprintf(ctx, "suffix token-string missing for field: '%s'", name); else if (tokenizer == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for unescaping token-string for field: '%s'", name); else if (uncopied_suffixes_str == NULL) ln_dbgprintf(ctx, "suffix-list missing for field: '%s'", name); else if (suffixes_str == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for suffix-list for field: '%s'", name); else if (pData->nsuffix == 0) ln_dbgprintf(ctx, "couldn't read suffix-value(s) for field: '%s'", name); else if (pData->suffix_offsets == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for suffix-list element references for field: " "'%s'", name); else if (pData->suffix_lengths == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for suffix-list element lengths for field: '%s'", name); else if (pData->suffixes_str == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for suffix-list for field: '%s'", name); else if (field_type == NULL) ln_dbgprintf(ctx, "field-type declaration missing for field: '%s'", name); else if (pData->ctx == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for normalizer-context for field: '%s'", name); suffixed_parser_data_destructor((void**)&pData); } free_pcons_args(&args); if (suffixes_str != NULL) free(suffixes_str); if (name != NULL) free(name); return pData; } void* suffixed_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx) { return _suffixed_parser_data_constructor(node, ctx, node->raw_data, NULL, NULL); } void* named_suffixed_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx) { int r = LN_BADCONFIG; pcons_args_t* args = NULL; char* name = NULL; const char* value_field_name = NULL; const char* suffix_field_name = NULL; const char* remaining_args = NULL; es_str_t* unnamed_suffix_args = NULL; struct suffixed_parser_data_s* pData = NULL; CHKN(name = es_str2cstr(node->name, NULL)); CHKN(args = pcons_args(node->raw_data, 3)); CHKN(value_field_name = pcons_arg(args, 0, NULL)); CHKN(suffix_field_name = pcons_arg(args, 1, NULL)); CHKN(remaining_args = pcons_arg(args, 2, NULL)); CHKN(unnamed_suffix_args = es_newStrFromCStr(remaining_args, strlen(remaining_args))); CHKN(pData = _suffixed_parser_data_constructor(node, ctx, unnamed_suffix_args, value_field_name, suffix_field_name)); r = 0; done: if (r != 0) { if (name == NULL) ln_dbgprintf(ctx, "couldn't allocate memory named_suffixed-field name"); else if (args == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for argument-parsing for field: %s", name); else if (value_field_name == NULL) ln_dbgprintf(ctx, "key-name for value not provided for field: %s", name); else if (suffix_field_name == NULL) ln_dbgprintf(ctx, "key-name for suffix not provided for field: %s", name); else if (unnamed_suffix_args == NULL) ln_dbgprintf(ctx, "couldn't allocate memory for unnamed-suffix-field args for field: %s", name); else if (pData == NULL) ln_dbgprintf(ctx, "couldn't create parser-data for field: %s", name); suffixed_parser_data_destructor((void**)&pData); } if (unnamed_suffix_args != NULL) free(unnamed_suffix_args); if (args != NULL) free_pcons_args(&args); if (name != NULL) free(name); return pData; } void suffixed_parser_data_destructor(void** dataPtr) { if ((*dataPtr) != NULL) { struct suffixed_parser_data_s *pData = (struct suffixed_parser_data_s*) *dataPtr; if (pData->suffixes_str != NULL) free(pData->suffixes_str); if (pData->suffix_offsets != NULL) free(pData->suffix_offsets); if (pData->suffix_lengths != NULL) free(pData->suffix_lengths); if (pData->value_field_name != NULL) free(pData->value_field_name); if (pData->suffix_field_name != NULL) free(pData->suffix_field_name); if (pData->ctx != NULL) ln_exitCtx(pData->ctx); free(pData); *dataPtr = NULL; } } /** * Just get everything till the end of string. */ PARSER(Rest) assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); /* silence the warning about unused variable */ (void)str; /* success, persist */ *parsed = strLen - *offs; r = 0; return r; } /** * Parse a possibly quoted string. In this initial implementation, escaping of the quote * char is not supported. A quoted string is one start starts with a double quote, * has some text (not containing double quotes) and ends with the first double * quote character seen. The extracted string does NOT include the quote characters. * swisskid, 2015-01-21 */ PARSER(OpQuotedString) const char *c; size_t i; char *cstr = NULL; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; if(c[i] != '"') { while(i < strLen && c[i] != ' ') i++; if(i == *offs) goto done; /* success, persist */ *parsed = i - *offs; /* create JSON value to save quoted string contents */ CHKN(cstr = strndup((char*)c + *offs, *parsed)); } else { ++i; /* search end of string */ while(i < strLen && c[i] != '"') i++; if(i == strLen || c[i] != '"') goto done; /* success, persist */ *parsed = i + 1 - *offs; /* "eat" terminal double quote */ /* create JSON value to save quoted string contents */ CHKN(cstr = strndup((char*)c + *offs + 1, *parsed - 2)); } CHKN(*value = json_object_new_string(cstr)); r = 0; /* success */ done: free(cstr); return r; } /** * Parse a quoted string. In this initial implementation, escaping of the quote * char is not supported. A quoted string is one start starts with a double quote, * has some text (not containing double quotes) and ends with the first double * quote character seen. The extracted string does NOT include the quote characters. * rgerhards, 2011-01-14 */ PARSER(QuotedString) const char *c; size_t i; char *cstr = NULL; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; if(i + 2 > strLen) goto done; /* needs at least 2 characters */ if(c[i] != '"') goto done; ++i; /* search end of string */ while(i < strLen && c[i] != '"') i++; if(i == strLen || c[i] != '"') goto done; /* success, persist */ *parsed = i + 1 - *offs; /* "eat" terminal double quote */ /* create JSON value to save quoted string contents */ CHKN(cstr = strndup((char*)c + *offs + 1, *parsed - 2)); CHKN(*value = json_object_new_string(cstr)); r = 0; /* success */ done: free(cstr); return r; } /** * Parse an ISO date, that is YYYY-MM-DD (exactly this format). * Note: we do manual loop unrolling -- this is fast AND efficient. * rgerhards, 2011-01-14 */ PARSER(ISODate) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; if(*offs+10 > strLen) goto done; /* if it is not 10 chars, it can't be an ISO date */ /* year */ if(!isdigit(c[i])) goto done; if(!isdigit(c[i+1])) goto done; if(!isdigit(c[i+2])) goto done; if(!isdigit(c[i+3])) goto done; if(c[i+4] != '-') goto done; /* month */ if(c[i+5] == '0') { if(c[i+6] < '1' || c[i+6] > '9') goto done; } else if(c[i+5] == '1') { if(c[i+6] < '0' || c[i+6] > '2') goto done; } else { goto done; } if(c[i+7] != '-') goto done; /* day */ if(c[i+8] == '0') { if(c[i+9] < '1' || c[i+9] > '9') goto done; } else if(c[i+8] == '1' || c[i+8] == '2') { if(!isdigit(c[i+9])) goto done; } else if(c[i+8] == '3') { if(c[i+9] != '0' && c[i+9] != '1') goto done; } else { goto done; } /* success, persist */ *parsed = 10; r = 0; /* success */ done: return r; } /** * Parse a Cisco interface spec. Sample for such a spec are: * outside:192.168.52.102/50349 * inside:192.168.1.15/56543 (192.168.1.112/54543) * outside:192.168.1.13/50179 (192.168.1.13/50179)(LOCAL\some.user) * outside:192.168.1.25/41850(LOCAL\RG-867G8-DEL88D879BBFFC8) * inside:192.168.1.25/53 (192.168.1.25/53) (some.user) * 192.168.1.15/0(LOCAL\RG-867G8-DEL88D879BBFFC8) * From this, we conclude the format is: * [interface:]ip/port [SP (ip2/port2)] [[SP](username)] * In order to match, this syntax must start on a non-whitespace char * other than colon. */ PARSER(CiscoInterfaceSpec) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; if(c[i] == ':' || isspace(c[i])) goto done; /* first, check if we have an interface. We do this by trying * to detect if we have an IP. If we have, obviously no interface * is present. Otherwise, we check if we have a valid interface. */ int bHaveInterface = 0; size_t idxInterface = 0; size_t lenInterface = 0; int bHaveIP = 0; size_t lenIP; size_t idxIP = i; if(ln_parseIPv4(str, strLen, &i, node, &lenIP, NULL) == 0) { bHaveIP = 1; i += lenIP - 1; /* position on delimiter */ } else { idxInterface = i; while(i < strLen) { if(isspace(c[i])) goto done; if(c[i] == ':') break; ++i; } lenInterface = i - idxInterface; bHaveInterface = 1; } if(i == strLen) goto done; ++i; /* skip over colon */ /* we now utilize our other parser helpers */ if(!bHaveIP) { idxIP = i; if(ln_parseIPv4(str, strLen, &i, node, &lenIP, NULL) != 0) goto done; i += lenIP; } if(i == strLen || c[i] != '/') goto done; ++i; /* skip slash */ const size_t idxPort = i; size_t lenPort; if(ln_parseNumber(str, strLen, &i, node, &lenPort, NULL) != 0) goto done; i += lenPort; if(i == strLen) goto success; /* check if optional second ip/port is present * We assume we must at least have 5 chars [" (::1)"] */ int bHaveIP2 = 0; size_t idxIP2 = 0, lenIP2 = 0; size_t idxPort2 = 0, lenPort2 = 0; if(i+5 < strLen && c[i] == ' ' && c[i+1] == '(') { size_t iTmp = i+2; /* skip over " (" */ idxIP2 = iTmp; if(ln_parseIPv4(str, strLen, &iTmp, node, &lenIP2, NULL) == 0) { iTmp += lenIP2; if(i < strLen || c[iTmp] == '/') { ++iTmp; /* skip slash */ idxPort2 = iTmp; if(ln_parseNumber(str, strLen, &iTmp, node, &lenPort2, NULL) == 0) { iTmp += lenPort2; if(iTmp < strLen && c[iTmp] == ')') { i = iTmp + 1; /* match, so use new index */ bHaveIP2 = 1; } } } } } /* check if optional username is present * We assume we must at least have 3 chars ["(n)"] */ int bHaveUser = 0; size_t idxUser = 0; size_t lenUser = 0; if( (i+2 < strLen && c[i] == '(' && !isspace(c[i+1]) ) || (i+3 < strLen && c[i] == ' ' && c[i+1] == '(' && !isspace(c[i+2])) ) { idxUser = i + ((c[i] == ' ') ? 2 : 1); /* skip [SP]'(' */ size_t iTmp = idxUser; while(iTmp < strLen && !isspace(c[iTmp]) && c[iTmp] != ')') ++iTmp; /* just scan */ if(iTmp < strLen && c[iTmp] == ')') { i = iTmp + 1; /* we have a match, so use new index */ bHaveUser = 1; lenUser = iTmp - idxUser; } } /* all done, save data */ if(value == NULL) goto success; CHKN(*value = json_object_new_object()); json_object *json; if(bHaveInterface) { CHKN(json = json_object_new_string_len(c+idxInterface, lenInterface)); json_object_object_add_ex(*value, "interface", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); } CHKN(json = json_object_new_string_len(c+idxIP, lenIP)); json_object_object_add_ex(*value, "ip", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); CHKN(json = json_object_new_string_len(c+idxPort, lenPort)); json_object_object_add_ex(*value, "port", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); if(bHaveIP2) { CHKN(json = json_object_new_string_len(c+idxIP2, lenIP2)); json_object_object_add_ex(*value, "ip2", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); CHKN(json = json_object_new_string_len(c+idxPort2, lenPort2)); json_object_object_add_ex(*value, "port2", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); } if(bHaveUser) { CHKN(json = json_object_new_string_len(c+idxUser, lenUser)); json_object_object_add_ex(*value, "user", json, JSON_C_OBJECT_ADD_KEY_IS_NEW|JSON_C_OBJECT_KEY_IS_CONSTANT); } success: /* success, persist */ *parsed = i - *offs; r = 0; /* success */ done: if(r != 0 && value != NULL && *value != NULL) { json_object_put(*value); *value = NULL; /* to be on the save side */ } return r; } /** * Parse a duration. A duration is similar to a timestamp, except that * it tells about time elapsed. As such, hours can be larger than 23 * and hours may also be specified by a single digit (this, for example, * is commonly done in Cisco software). * Note: we do manual loop unrolling -- this is fast AND efficient. */ PARSER(Duration) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; /* hour is a bit tricky */ if(!isdigit(c[i])) goto done; ++i; if(isdigit(c[i])) ++i; if(c[i] == ':') ++i; else goto done; if(i+5 > strLen) goto done;/* if it is not 5 chars from here, it can't be us */ if(c[i] < '0' || c[i] > '5') goto done; if(!isdigit(c[i+1])) goto done; if(c[i+2] != ':') goto done; if(c[i+3] < '0' || c[i+3] > '5') goto done; if(!isdigit(c[i+4])) goto done; /* success, persist */ *parsed = (i + 5) - *offs; r = 0; /* success */ done: return r; } /** * Parse a timestamp in 24hr format (exactly HH:MM:SS). * Note: we do manual loop unrolling -- this is fast AND efficient. * rgerhards, 2011-01-14 */ PARSER(Time24hr) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; if(*offs+8 > strLen) goto done; /* if it is not 8 chars, it can't be us */ /* hour */ if(c[i] == '0' || c[i] == '1') { if(!isdigit(c[i+1])) goto done; } else if(c[i] == '2') { if(c[i+1] < '0' || c[i+1] > '3') goto done; } else { goto done; } /* TODO: the code below is a duplicate of 24hr parser - create common function */ if(c[i+2] != ':') goto done; if(c[i+3] < '0' || c[i+3] > '5') goto done; if(!isdigit(c[i+4])) goto done; if(c[i+5] != ':') goto done; if(c[i+6] < '0' || c[i+6] > '5') goto done; if(!isdigit(c[i+7])) goto done; /* success, persist */ *parsed = 8; r = 0; /* success */ done: return r; } /** * Parse a timestamp in 12hr format (exactly HH:MM:SS). * Note: we do manual loop unrolling -- this is fast AND efficient. * TODO: the code below is a duplicate of 24hr parser - create common function? * rgerhards, 2011-01-14 */ PARSER(Time12hr) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); c = str; i = *offs; if(*offs+8 > strLen) goto done; /* if it is not 8 chars, it can't be us */ /* hour */ if(c[i] == '0') { if(!isdigit(c[i+1])) goto done; } else if(c[i] == '1') { if(c[i+1] < '0' || c[i+1] > '2') goto done; } else { goto done; } if(c[i+2] != ':') goto done; if(c[i+3] < '0' || c[i+3] > '5') goto done; if(!isdigit(c[i+4])) goto done; if(c[i+5] != ':') goto done; if(c[i+6] < '0' || c[i+6] > '5') goto done; if(!isdigit(c[i+7])) goto done; /* success, persist */ *parsed = 8; r = 0; /* success */ done: return r; } /* helper to IPv4 address parser, checks the next set of numbers. * Syntax 1 to 3 digits, value together not larger than 255. * @param[in] str parse buffer * @param[in/out] offs offset into buffer, updated if successful * @return 0 if OK, 1 otherwise */ static int chkIPv4AddrByte(const char *str, size_t strLen, size_t *offs) { int val = 0; int r = 1; /* default: done -- simplifies things */ const char *c; size_t i = *offs; c = str; if(i == strLen || !isdigit(c[i])) goto done; val = c[i++] - '0'; if(i < strLen && isdigit(c[i])) { val = val * 10 + c[i++] - '0'; if(i < strLen && isdigit(c[i])) val = val * 10 + c[i++] - '0'; } if(val > 255) /* cannot be a valid IP address byte! */ goto done; *offs = i; r = 0; done: return r; } /** * Parser for IPv4 addresses. */ PARSER(IPv4) const char *c; size_t i; assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); i = *offs; if(i + 7 > strLen) { /* IPv4 addr requires at least 7 characters */ goto done; } c = str; /* byte 1*/ if(chkIPv4AddrByte(str, strLen, &i) != 0) goto done; if(i == strLen || c[i++] != '.') goto done; /* byte 2*/ if(chkIPv4AddrByte(str, strLen, &i) != 0) goto done; if(i == strLen || c[i++] != '.') goto done; /* byte 3*/ if(chkIPv4AddrByte(str, strLen, &i) != 0) goto done; if(i == strLen || c[i++] != '.') goto done; /* byte 4 - we do NOT need any char behind it! */ if(chkIPv4AddrByte(str, strLen, &i) != 0) goto done; /* if we reach this point, we found a valid IP address */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /* skip past the IPv6 address block, parse pointer is set to * first char after the block. Returns an error if already at end * of string. * @param[in] str parse buffer * @param[in/out] offs offset into buffer, updated if successful * @return 0 if OK, 1 otherwise */ static int skipIPv6AddrBlock(const char *const __restrict__ str, const size_t strLen, size_t *const __restrict__ offs) { int j; if(*offs == strLen) return 1; for(j = 0 ; j < 4 && *offs+j < strLen && isxdigit(str[*offs+j]) ; ++j) /*just skip*/ ; *offs += j; return 0; } /** * Parser for IPv6 addresses. * Bases on RFC4291 Section 2.2. The address must be followed * by whitespace or end-of-string, else it is not considered * a valid address. This prevents false positives. */ PARSER(IPv6) const char *c; size_t i; size_t beginBlock; /* last block begin in case we need IPv4 parsing */ int hasIPv4 = 0; int nBlocks = 0; /* how many blocks did we already have? */ int bHad0Abbrev = 0; /* :: already used? */ assert(str != NULL); assert(offs != NULL); assert(parsed != NULL); i = *offs; if(i + 2 > strLen) { /* IPv6 addr requires at least 2 characters ("::") */ goto done; } c = str; /* check that first block is non-empty */ if(! ( isxdigit(c[i]) || (c[i] == ':' && c[i+1] == ':') ) ) goto done; /* try for all potential blocks plus one more (so we see errors!) */ for(int j = 0 ; j < 9 ; ++j) { beginBlock = i; if(skipIPv6AddrBlock(str, strLen, &i) != 0) goto done; nBlocks++; if(i == strLen) goto chk_ok; if(isspace(c[i])) goto chk_ok; if(c[i] == '.'){ /* IPv4 processing! */ hasIPv4 = 1; break; } if(c[i] != ':') goto done; i++; /* "eat" ':' */ if(i == strLen) goto chk_ok; /* check for :: */ if(bHad0Abbrev) { if(c[i] == ':') goto done; } else { if(c[i] == ':') { bHad0Abbrev = 1; ++i; if(i == strLen) goto chk_ok; } } } if(hasIPv4) { size_t ipv4_parsed; --nBlocks; /* prevent pure IPv4 address to be recognized */ if(beginBlock == *offs) goto done; i = beginBlock; if(ln_parseIPv4(str, strLen, &i, node, &ipv4_parsed, NULL) != 0) goto done; i += ipv4_parsed; } chk_ok: /* we are finished parsing, check if things are ok */ if(nBlocks > 8) goto done; if(bHad0Abbrev && nBlocks >= 8) goto done; /* now check if trailing block is missing. Note that i is already * on next character, so we need to go two back. Two are always * present, else we would not reach this code here. */ if(c[i-1] == ':' && c[i-2] != ':') goto done; /* if we reach this point, we found a valid IP address */ *parsed = i - *offs; r = 0; /* success */ done: return r; } /* check if a char is valid inside a name of the iptables motif. * We try to keep the set as slim as possible, because the iptables * parser may otherwise create a very broad match (especially the * inclusion of simple words like "DF" cause grief here). * Note: we have taken the permitted set from iptables log samples. * Report bugs if we missed some additional rules. */ static inline int isValidIPTablesNameChar(const char c) { /* right now, upper case only is valid */ return ('A' <= c && c <= 'Z') ? 1 : 0; } /* helper to iptables parser, parses out a a single name=value pair */ static int parseIPTablesNameValue(const char *const __restrict__ str, const size_t strLen, size_t *const __restrict__ offs, struct json_object *const __restrict__ valroot) { int r = LN_WRONGPARSER; size_t i = *offs; char *name = NULL; const size_t iName = i; while(i < strLen && isValidIPTablesNameChar(str[i])) ++i; if(i == iName || (i < strLen && str[i] != '=' && str[i] != ' ')) goto done; /* no name at all! */ const ssize_t lenName = i - iName; ssize_t iVal = -1; size_t lenVal = i - iVal; if(i < strLen && str[i] != ' ') { /* we have a real value (not just a flag name like "DF") */ ++i; /* skip '=' */ iVal = i; while(i < strLen && !isspace(str[i])) ++i; lenVal = i - iVal; } /* parsing OK */ *offs = i; r = 0; if(valroot == NULL) goto done; CHKN(name = malloc(lenName+1)); memcpy(name, str+iName, lenName); name[lenName] = '\0'; json_object *json; if(iVal == -1) { json = NULL; } else { CHKN(json = json_object_new_string_len(str+iVal, lenVal)); } json_object_object_add(valroot, name, json); done: free(name); return r; } /** * Parser for iptables logs (the structured part). * This parser is named "v2-iptables" because of a traditional * parser named "iptables", which we do not want to replace, at * least right now (we may re-think this before the first release). * For performance reasons, this works in two stages. In the first * stage, we only detect if the motif is correct. The second stage is * only called when we know it is. In it, we go once again over the * message again and actually extract the data. This is done because * data extraction is relatively expensive and in most cases we will * have much more frequent mismatches than matches. * Note that this motif must have at least one field, otherwise it * could detect things that are not iptables to be it. Further limits * may be imposed in the future as we see additional need. * added 2015-04-30 rgerhards */ PARSER(v2IPTables) size_t i = *offs; int nfields = 0; /* stage one */ while(i < strLen) { CHKR(parseIPTablesNameValue(str, strLen, &i, NULL)); ++nfields; /* exactly one SP is permitted between fields */ if(i < strLen && str[i] == ' ') ++i; } if(nfields < 2) { FAIL(LN_WRONGPARSER); } /* success, persist */ *parsed = i - *offs; r = 0; /* stage two */ if(value == NULL) goto done; i = *offs; CHKN(*value = json_object_new_object()); while(i < strLen) { CHKR(parseIPTablesNameValue(str, strLen, &i, *value)); while(i < strLen && isspace(str[i])) ++i; } done: if(r != 0 && value != NULL && *value != NULL) { json_object_put(*value); *value = NULL; } return r; } /** * Parse JSON. This parser tries to find JSON data inside a message. * If it finds valid JSON, it will extract it. Extra data after the * JSON is permitted. * Note: the json-c JSON parser treats whitespace after the actual * json to be part of the json. So in essence, any whitespace is * processed by this parser. We use the same semantics to keep things * neatly in sync. If json-c changes for some reason or we switch to * an alternate json lib, we probably need to be sure to keep that * behaviour, and probably emulate it. * added 2015-04-28 by rgerhards, v1.1.2 */ PARSER(JSON) const size_t i = *offs; struct json_tokener *tokener = NULL; if(str[i] != '{' && str[i] != ']') { /* this can't be json, see RFC4627, Sect. 2 * see this bug in json-c: * https://github.com/json-c/json-c/issues/181 * In any case, it's better to do this quick check, * even if json-c did not have the bug because this * check here is much faster than calling the parser. */ goto done; } if((tokener = json_tokener_new()) == NULL) goto done; struct json_object *const json = json_tokener_parse_ex(tokener, str+i, (int) (strLen - i)); if(json == NULL) goto done; /* success, persist */ *parsed = (i + tokener->char_offset) - *offs; r = 0; /* success */ if(value == NULL) { json_object_put(json); } else { *value = json; } done: if(tokener != NULL) json_tokener_free(tokener); return r; } /* check if a char is valid inside a name of a NameValue list * The set of valid characters may be extended if there is good * need to do so. We have selected the current set carefully, but * may have overlooked some cases. */ static inline int isValidNameChar(const char c) { return (isalnum(c) || c == '.' || c == '_' || c == '-' ) ? 1 : 0; } /* helper to NameValue parser, parses out a a single name=value pair * * name must be alphanumeric characters, value must be non-whitespace * characters, if quoted than with symmetric quotes. Supported formats * - name=value * - name="value" * - name='value' * Note "name=" is valid and means a field with empty value. * TODO: so far, quote characters are not permitted WITHIN quoted values. */ static int parseNameValue(const char *const __restrict__ str, const size_t strLen, size_t *const __restrict__ offs, struct json_object *const __restrict__ valroot) { int r = LN_WRONGPARSER; size_t i = *offs; char *name = NULL; const size_t iName = i; while(i < strLen && isValidNameChar(str[i])) ++i; if(i == iName || str[i] != '=') goto done; /* no name at all! */ const size_t lenName = i - iName; ++i; /* skip '=' */ const size_t iVal = i; while(i < strLen && !isspace(str[i])) ++i; const size_t lenVal = i - iVal; /* parsing OK */ *offs = i; r = 0; if(valroot == NULL) goto done; CHKN(name = malloc(lenName+1)); memcpy(name, str+iName, lenName); name[lenName] = '\0'; json_object *json; CHKN(json = json_object_new_string_len(str+iVal, lenVal)); json_object_object_add(valroot, name, json); done: free(name); return r; } /** * Parse CEE syslog. * This essentially is a JSON parser, with additional restrictions: * The message must start with "@cee:" and json must immediately follow (whitespace permitted). * after the JSON, there must be no other non-whitespace characters. * In other words: the message must consist of a single JSON object, * only. * added 2015-04-28 by rgerhards, v1.1.2 */ PARSER(CEESyslog) size_t i = *offs; struct json_tokener *tokener = NULL; struct json_object *json = NULL; if(strLen < i + 7 || /* "@cee:{}" is minimum text */ str[i] != '@' || str[i+1] != 'c' || str[i+2] != 'e' || str[i+3] != 'e' || str[i+4] != ':') goto done; /* skip whitespace */ for(i += 5 ; i < strLen && isspace(str[i]) ; ++i) /* just skip */; if(i == strLen || str[i] != '{') goto done; /* note: we do not permit arrays in CEE mode */ if((tokener = json_tokener_new()) == NULL) goto done; json = json_tokener_parse_ex(tokener, str+i, (int) (strLen - i)); if(json == NULL) goto done; if(i + tokener->char_offset != strLen) goto done; /* success, persist */ *parsed = strLen; r = 0; /* success */ if(value != NULL) { *value = json; json = NULL; /* do NOT free below! */ } done: if(tokener != NULL) json_tokener_free(tokener); if(json != NULL) json_object_put(json); return r; } /** * Parser for name/value pairs. * On entry must point to alnum char. All following chars must be * name/value pairs delimited by whitespace up until the end of string. * For performance reasons, this works in two stages. In the first * stage, we only detect if the motif is correct. The second stage is * only called when we know it is. In it, we go once again over the * message again and actually extract the data. This is done because * data extraction is relatively expensive and in most cases we will * have much more frequent mismatches than matches. * added 2015-04-25 rgerhards */ PARSER(NameValue) size_t i = *offs; /* stage one */ while(i < strLen) { CHKR(parseNameValue(str, strLen, &i, NULL)); while(i < strLen && isspace(str[i])) ++i; } /* success, persist */ *parsed = i - *offs; r = 0; /* success */ /* stage two */ if(value == NULL) goto done; i = *offs; CHKN(*value = json_object_new_object()); while(i < strLen) { CHKR(parseNameValue(str, strLen, &i, *value)); while(i < strLen && isspace(str[i])) ++i; } /* TODO: fix mem leak if alloc json fails */ done: return r; } /** * Parse a MAC layer address. * The standard (IEEE 802) format for printing MAC-48 addresses in * human-friendly form is six groups of two hexadecimal digits, * separated by hyphens (-) or colons (:), in transmission order * (e.g. 01-23-45-67-89-ab or 01:23:45:67:89:ab ). * This form is also commonly used for EUI-64. * from: http://en.wikipedia.org/wiki/MAC_address * * This parser must start on a hex digit. * added 2015-05-04 by rgerhards, v1.1.2 */ PARSER(MAC48) size_t i = *offs; char delim; if(strLen < i + 17 || /* this motif has exactly 17 characters */ !isxdigit(str[i]) || !isxdigit(str[i+1]) ) FAIL(LN_WRONGPARSER); if(str[i+2] == ':') delim = ':'; else if(str[i+2] == '-') delim = '-'; else FAIL(LN_WRONGPARSER); /* first byte ok */ if(!isxdigit(str[i+3]) || !isxdigit(str[i+4]) || str[i+5] != delim || /* 2nd byte ok */ !isxdigit(str[i+6]) || !isxdigit(str[i+7]) || str[i+8] != delim || /* 3rd byte ok */ !isxdigit(str[i+9]) || !isxdigit(str[i+10]) || str[i+11] != delim || /* 4th byte ok */ !isxdigit(str[i+12]) || !isxdigit(str[i+13]) || str[i+14] != delim || /* 5th byte ok */ !isxdigit(str[i+15]) || !isxdigit(str[i+16]) /* 6th byte ok */ ) FAIL(LN_WRONGPARSER); /* success, persist */ *parsed = 17; r = 0; /* success */ if(value != NULL) { CHKN(*value = json_object_new_string_len(str+i, 17)); } done: return r; } /* This parses the extension value and updates the index * to point to the end of it. */ static int cefParseExtensionValue(const char *const __restrict__ str, const size_t strLen, size_t *__restrict__ iEndVal) { int r = 0; size_t i = *iEndVal; size_t iLastWordBegin; /* first find next unquoted equal sign and record begin of * last word in front of it - this is the actual end of the * current name/value pair and the begin of the next one. */ int hadSP = 0; int inEscape = 0; for(iLastWordBegin = 0 ; i < strLen ; ++i) { if(inEscape) { if(str[i] != '=' && str[i] != '\\' && str[i] != 'r' && str[i] != 'n') FAIL(LN_WRONGPARSER); inEscape = 0; } else { if(str[i] == '=') { break; } else if(str[i] == '\\') { inEscape = 1; } else if(str[i] == ' ') { hadSP = 1; } else { if(hadSP) { iLastWordBegin = i; hadSP = 0; } } } } /* Note: iLastWordBegin can never be at offset zero, because * the CEF header starts there! */ if(i < strLen) { *iEndVal = (iLastWordBegin == 0) ? i : iLastWordBegin - 1; } else { *iEndVal = i; } done: return r; } /* must be positioned on first char of name, returns index * of end of name. * Note: ArcSight violates the CEF spec ifself: they generate * leading underscores in their extension names, which are * definitely not alphanumeric. We still accept them... * They also seem to use dots. */ static int cefParseName(const char *const __restrict__ str, const size_t strLen, size_t *const __restrict__ i) { int r = 0; while(*i < strLen && str[*i] != '=') { if(!(isalnum(str[*i]) || str[*i] == '_' || str[*i] == '.')) FAIL(LN_WRONGPARSER); ++(*i); } done: return r; } /* parse CEF extensions. They are basically name=value * pairs with the ugly exception that values may contain * spaces but need NOT to be quoted. Thankfully, at least * names are specified as being alphanumeric without spaces * in them. So we must add a lookahead parser to check if * a word is a name (and thus the begin of a new pair) or * not. This is done by subroutines. */ static int cefParseExtensions(const char *const __restrict__ str, const size_t strLen, size_t *const __restrict__ offs, json_object *const __restrict__ jroot) { int r = 0; size_t i = *offs; size_t iName, lenName; size_t iValue, lenValue; char *name = NULL; char *value = NULL; while(i < strLen) { while(i < strLen && str[i] == ' ') ++i; iName = i; CHKR(cefParseName(str, strLen, &i)); if(i+1 >= strLen || str[i] != '=') FAIL(LN_WRONGPARSER); lenName = i - iName; ++i; /* skip '=' */ iValue = i; CHKR(cefParseExtensionValue(str, strLen, &i)); lenValue = i - iValue; ++i; /* skip past value */ if(jroot != NULL) { CHKN(name = malloc(sizeof(char) * (lenName + 1))); memcpy(name, str+iName, lenName); name[lenName] = '\0'; CHKN(value = malloc(sizeof(char) * (lenValue + 1))); /* copy value but escape it */ size_t iDst = 0; for(size_t iSrc = 0 ; iSrc < lenValue ; ++iSrc) { if(str[iValue+iSrc] == '\\') { ++iSrc; /* we know the next char must exist! */ switch(str[iValue+iSrc]) { case '=': value[iDst] = '='; break; case 'n': value[iDst] = '\n'; break; case 'r': value[iDst] = '\r'; break; case '\\': value[iDst] = '\\'; break; default: break; } } else { value[iDst] = str[iValue+iSrc]; } ++iDst; } value[iDst] = '\0'; json_object *json; CHKN(json = json_object_new_string(value)); json_object_object_add(jroot, name, json); free(name); name = NULL; free(value); value = NULL; } } done: free(name); free(value); return r; } /* gets a CEF header field. Must be positioned on the * first char after the '|' in front of field. * Note that '|' may be escaped as "\|", which also means * we need to supprot "\\" (see CEF spec for details). * We return the string in *val, if val is non-null. In * that case we allocate memory that the caller must free. * This is necessary because there are potentially escape * sequences inside the string. */ static int cefGetHdrField(const char *const __restrict__ str, const size_t strLen, size_t *const __restrict__ offs, char **val) { int r = 0; size_t i = *offs; assert(str[i] != '|'); while(i < strLen && str[i] != '|') { if(str[i] == '\\') { ++i; /* skip esc char */ if(str[i] != '\\' && str[i] != '|') FAIL(LN_WRONGPARSER); } ++i; /* scan to next delimiter */ } if(str[i] != '|') FAIL(LN_WRONGPARSER); const size_t iBegin = *offs; /* success, persist */ *offs = i + 1; if(val == NULL) { r = 0; goto done; } const size_t len = i - iBegin; CHKN(*val = malloc(len + 1)); size_t iDst = 0; for(size_t iSrc = 0 ; iSrc < len ; ++iSrc) { if(str[iBegin+iSrc] == '\\') ++iSrc; /* we already checked above that this is OK! */ (*val)[iDst++] = str[iBegin+iSrc]; } (*val)[iDst] = 0; r = 0; done: return r; } /** * Parser for ArcSight Common Event Format (CEF) version 0. * added 2015-05-05 by rgerhards, v1.1.2 */ PARSER(CEF) size_t i = *offs; char *vendor = NULL; char *product = NULL; char *version = NULL; char *sigID = NULL; char *name = NULL; char *severity = NULL; /* minimum header: "CEF:0|x|x|x|x|x|x|" --> 17 chars */ if(strLen < i + 17 || str[i] != 'C' || str[i+1] != 'E' || str[i+2] != 'F' || str[i+3] != ':' || str[i+4] != '0' || str[i+5] != '|' ) FAIL(LN_WRONGPARSER); i += 6; /* position on '|' */ CHKR(cefGetHdrField(str, strLen, &i, (value == NULL) ? NULL : &vendor)); CHKR(cefGetHdrField(str, strLen, &i, (value == NULL) ? NULL : &product)); CHKR(cefGetHdrField(str, strLen, &i, (value == NULL) ? NULL : &version)); CHKR(cefGetHdrField(str, strLen, &i, (value == NULL) ? NULL : &sigID)); CHKR(cefGetHdrField(str, strLen, &i, (value == NULL) ? NULL : &name)); CHKR(cefGetHdrField(str, strLen, &i, (value == NULL) ? NULL : &severity)); ++i; /* skip over terminal '|' */ /* OK, we now know we have a good header. Now, we need * to process extensions. * This time, we do NOT pre-process the extension, but rather * persist them directly to JSON. This is contrary to other * parsers, but as the CEF header is pretty unique, this time * it is extremely unlikely we will get a no-match during * extension processing. Even if so, nothing bad happens, as * the extracted data is discarded. But the regular case saves * us processing time and complexity. The only time when we * cannot directly process it is when the caller asks us not * to persist the data. So this must be handled differently. */ size_t iBeginExtensions = i; CHKR(cefParseExtensions(str, strLen, &i, NULL)); /* success, persist */ *parsed = *offs - i; r = 0; /* success */ if(value != NULL) { CHKN(*value = json_object_new_object()); json_object *json; CHKN(json = json_object_new_string(vendor)); json_object_object_add(*value, "DeviceVendor", json); CHKN(json = json_object_new_string(product)); json_object_object_add(*value, "DeviceProduct", json); CHKN(json = json_object_new_string(version)); json_object_object_add(*value, "DeviceVersion", json); CHKN(json = json_object_new_string(sigID)); json_object_object_add(*value, "SignatureID", json); CHKN(json = json_object_new_string(name)); json_object_object_add(*value, "Name", json); CHKN(json = json_object_new_string(severity)); json_object_object_add(*value, "Severity", json); json_object *jext; CHKN(jext = json_object_new_object()); json_object_object_add(*value, "Extensions", jext); i = iBeginExtensions; cefParseExtensions(str, strLen, &i, jext); } done: if(r != 0 && value != NULL && *value != NULL) { json_object_put(*value); value = NULL; } free(vendor); free(product); free(version); free(sigID); free(name); free(severity); return r; } /** * Parser for Checkpoint LEA on-disk format. * added 2015-06-18 by rgerhards, v1.1.2 */ PARSER(CheckpointLEA) size_t i = *offs; size_t iName, lenName; size_t iValue, lenValue; int foundFields = 0; char *name = NULL; char *val = NULL; while(i < strLen) { while(i < strLen && str[i] == ' ') /* skip leading SP */ ++i; if(i == strLen) { /* OK if just trailing space */ if(foundFields == 0) FAIL(LN_WRONGPARSER); break; /* we are done with the loop, all processed */ } else { ++foundFields; } iName = i; /* TODO: do a stricter check? ... but we don't have a spec */ while(i < strLen && str[i] != ':') { ++i; } if(i+1 >= strLen || str[i] != ':') FAIL(LN_WRONGPARSER); lenName = i - iName; ++i; /* skip ':' */ while(i < strLen && str[i] == ' ') /* skip leading SP */ ++i; iValue = i; while(i < strLen && str[i] != ';') { ++i; } if(i+1 > strLen || str[i] != ';') FAIL(LN_WRONGPARSER); lenValue = i - iValue; ++i; /* skip ';' */ if(value != NULL) { CHKN(name = malloc(sizeof(char) * (lenName + 1))); memcpy(name, str+iName, lenName); name[lenName] = '\0'; CHKN(val = malloc(sizeof(char) * (lenValue + 1))); memcpy(val, str+iValue, lenValue); val[lenValue] = '\0'; if(*value == NULL) CHKN(*value = json_object_new_object()); json_object *json; CHKN(json = json_object_new_string(val)); json_object_object_add(*value, name, json); free(name); name = NULL; free(val); val = NULL; } } /* success, persist */ *parsed = *offs - i; r = 0; /* success */ done: free(name); free(val); if(r != 0 && value != NULL && *value != NULL) { json_object_put(*value); value = NULL; } return r; } liblognorm-2.0.8/src/v1_parser.h000066400000000000000000000212331511425433100165270ustar00rootroot00000000000000/* * liblognorm - a fast samples-based log normalization library * Copyright 2010-2015 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef LIBLOGNORM_V1_PARSER_H_INCLUDED #define LIBLOGNORM_V1_PARSER_H_INCLUDED #include "v1_ptree.h" /** * Parser interface * @param[in] str the to-be-parsed string * @param[in] strLen length of the to-be-parsed string * @param[in] offs an offset into the string * @param[in] node fieldlist with additional data; for simple * parsers, this sets variable "ed", which just is * string data. * @param[out] parsed bytes * @param[out] json object containing parsed data (can be unused) * @return 0 on success, something else otherwise */ /** * Parser for RFC5424 date. */ int ln_parseRFC5424Date(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for RFC3164 date. */ int ln_parseRFC3164Date(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for numbers. */ int ln_parseNumber(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for real-number in floating-pt representation */ int ln_parseFloat(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for hex numbers. */ int ln_parseHexNumber(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for kernel timestamps. */ int ln_parseKernelTimestamp(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for whitespace */ int ln_parseWhitespace(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for Words (SP-terminated strings). */ int ln_parseWord(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse everything up to a specific string. */ int ln_parseStringTo(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for Alphabetic words (no numbers, punct, ctrl, space). */ int ln_parseAlpha(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse everything up to a specific character. */ int ln_parseCharTo(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse everything up to a specific character (relaxed constraints, suitable for CSV) */ int ln_parseCharSeparated(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Get everything till the rest of string. */ int ln_parseRest(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse an optionally quoted string. */ int ln_parseOpQuotedString(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse a quoted string. */ int ln_parseQuotedString(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse an ISO date. */ int ln_parseISODate(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse a timestamp in 12hr format. */ int ln_parseTime12hr(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse a timestamp in 24hr format. */ int ln_parseTime24hr(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse a duration. */ int ln_parseDuration(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for IPv4 addresses. */ int ln_parseIPv4(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for IPv6 addresses. */ int ln_parseIPv6(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse JSON. */ int ln_parseJSON(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse cee syslog. */ int ln_parseCEESyslog(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parse iptables log, the new way */ int ln_parsev2IPTables(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser Cisco interface specifiers */ int ln_parseCiscoInterfaceSpec(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser 48 bit MAC layer addresses. */ int ln_parseMAC48(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for CEF version 0. */ int ln_parseCEF(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for Checkpoint LEA. */ int ln_parseCheckpointLEA(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Parser for name/value pairs. */ int ln_parseNameValue(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); /** * Get all tokens separated by tokenizer-string as array. */ int ln_parseTokenized(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); void* tokenized_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx); void tokenized_parser_data_destructor(void** dataPtr); #ifdef FEATURE_REGEXP /** * Get field matching regex */ int ln_parseRegex(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); void* regex_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx); void regex_parser_data_destructor(void** dataPtr); #endif /** * Match using the 'current' or 'separate rulebase' all over again from current match position */ int ln_parseRecursive(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); void* recursive_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx); void* descent_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx); void recursive_parser_data_destructor(void** dataPtr); /** * Get interpreted field */ int ln_parseInterpret(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); void* interpret_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx); void interpret_parser_data_destructor(void** dataPtr); /** * Parse a suffixed field */ int ln_parseSuffixed(const char *str, size_t strlen, size_t *offs, const ln_fieldList_t *node, size_t *parsed, struct json_object **value); void* suffixed_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx); void* named_suffixed_parser_data_constructor(ln_fieldList_t *node, ln_ctx ctx); void suffixed_parser_data_destructor(void** dataPtr); #endif /* #ifndef LIBLOGNORM_V1_PARSER_H_INCLUDED */ liblognorm-2.0.8/src/v1_ptree.c000066400000000000000000000622231511425433100163510ustar00rootroot00000000000000/** * @file ptree.c * @brief Implementation of the parse tree object. * @class ln_ptree ptree.h *//* * Copyright 2010 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include #include #include #include #include #define LOGNORM_V1_SUBSYSTEM /* indicate we are old cruft */ #include "v1_liblognorm.h" #include "annot.h" #include "internal.h" #include "lognorm.h" #include "v1_ptree.h" #include "v1_samp.h" #include "v1_parser.h" /** * Get base addr of common prefix. Takes length of prefix in account * and selects the right buffer. */ static inline unsigned char* prefixBase(struct ln_ptree *tree) { return (tree->lenPrefix <= sizeof(tree->prefix)) ? tree->prefix.data : tree->prefix.ptr; } struct ln_ptree* ln_newPTree(ln_ctx ctx, struct ln_ptree **parentptr) { struct ln_ptree *tree; if((tree = calloc(1, sizeof(struct ln_ptree))) == NULL) goto done; tree->parentptr = parentptr; tree->ctx = ctx; ctx->nNodes++; done: return tree; } void ln_deletePTreeNode(ln_fieldList_t *node) { ln_deletePTree(node->subtree); es_deleteStr(node->name); if(node->data != NULL) es_deleteStr(node->data); if(node->raw_data != NULL) es_deleteStr(node->raw_data); if(node->parser_data != NULL && node->parser_data_destructor != NULL) node->parser_data_destructor(&(node->parser_data)); free(node); } void ln_deletePTree(struct ln_ptree *tree) { ln_fieldList_t *node, *nextnode; size_t i; if(tree == NULL) goto done; if(tree->tags != NULL) json_object_put(tree->tags); for(node = tree->froot; node != NULL; node = nextnode) { nextnode = node->next; ln_deletePTreeNode(node); } /* need to free a large prefix buffer? */ if(tree->lenPrefix > sizeof(tree->prefix)) free(tree->prefix.ptr); for(i = 0 ; i < 256 ; ++i) if(tree->subtree[i] != NULL) ln_deletePTree(tree->subtree[i]); free(tree); done: return; } /** * Set the common prefix inside a note, taking into account the subtle * issues associated with it. * @return 0 on success, something else otherwise */ static int setPrefix(struct ln_ptree *tree, unsigned char *buf, size_t lenBuf, size_t offs) { int r; LN_DBGPRINTF(tree->ctx, "setPrefix lenBuf %zu, offs %zu", lenBuf, offs); tree->lenPrefix = lenBuf - offs; if(tree->lenPrefix > sizeof(tree->prefix)) { /* too-large for standard buffer, need to alloc one */ if((tree->prefix.ptr = malloc(tree->lenPrefix * sizeof(unsigned char))) == NULL) { r = LN_NOMEM; goto done; /* fail! */ } memcpy(tree->prefix.ptr, buf, tree->lenPrefix); } else { /* note: r->lenPrefix may be 0, but that is OK */ memcpy(tree->prefix.data, buf, tree->lenPrefix); } r = 0; done: return r; } /** * Check if the provided tree is a leaf. This means that it * does not contain any subtrees. * @return 1 if it is a leaf, 0 otherwise */ static int isLeaf(struct ln_ptree *tree) { int r = 0; int i; if(tree->froot != NULL) goto done; for(i = 0 ; i < 256 ; ++i) { if(tree->subtree[i] != NULL) goto done; } r = 1; done: return r; } /** * Check if the provided tree is a true leaf. This means that it * does not contain any subtrees of any kind and no prefix, * and it is not terminal leaf. * @return 1 if it is a leaf, 0 otherwise */ static inline int isTrueLeaf(struct ln_ptree *tree) { return((tree->lenPrefix == 0) && isLeaf(tree)) && !tree->flags.isTerminal; } struct ln_ptree * ln_addPTree(struct ln_ptree *tree, es_str_t *str, size_t offs) { struct ln_ptree *r; struct ln_ptree **parentptr; /**< pointer in parent that needs to be updated */ LN_DBGPRINTF(tree->ctx, "addPTree: offs %zu", offs); parentptr = &(tree->subtree[es_getBufAddr(str)[offs]]); /* First check if tree node is totally empty. If so, we can simply add * the prefix to this node. This case is important, because it happens * every time with a new field. */ if(isTrueLeaf(tree)) { if(setPrefix(tree, es_getBufAddr(str), es_strlen(str), offs) != 0) { r = NULL; } else { r = tree; } goto done; } if(tree->ctx->debug) { char *cstr = es_str2cstr(str, NULL); LN_DBGPRINTF(tree->ctx, "addPTree: add '%s', offs %zu, tree %p", cstr + offs, offs, tree); free(cstr); } if((r = ln_newPTree(tree->ctx, parentptr)) == NULL) goto done; if(setPrefix(r, es_getBufAddr(str) + offs + 1, es_strlen(str) - offs - 1, 0) != 0) { free(r); r = NULL; goto done; } *parentptr = r; done: return r; } /** * Split the provided tree (node) into two at the provided index into its * common prefix. This function exists to support splitting nodes when * a mismatch in the common prefix requires that. This function more or less * keeps the tree as it is, just changes the structure. No new node is added. * Usually, it is desired to add a new node. This must be made afterwards. * Note that we need to create a new tree *in front of* the current one, as * the current one contains field etc. subtree pointers. * @param[in] tree tree to split * @param[in] offs offset into common prefix (must be less than prefix length!) */ static struct ln_ptree* splitTree(struct ln_ptree *tree, unsigned short offs) { unsigned char *c; struct ln_ptree *r; unsigned short newlen; ln_ptree **newparentptr; /**< pointer in parent that needs to be updated */ assert(offs < tree->lenPrefix); if((r = ln_newPTree(tree->ctx, tree->parentptr)) == NULL) goto done; LN_DBGPRINTF(tree->ctx, "splitTree %p at offs %u", tree, offs); /* note: the overall prefix is reduced by one char, which is now taken * care of inside the "branch table". */ c = prefixBase(tree); //LN_DBGPRINTF(tree->ctx, "splitTree new bb, *(c+offs): '%s'", c); if(setPrefix(r, c, offs, 0) != 0) { ln_deletePTree(r); r = NULL; goto done; /* fail! */ } LN_DBGPRINTF(tree->ctx, "splitTree new tree %p lenPrefix=%u, char '%c'", r, r->lenPrefix, r->prefix.data[0]); /* add the proper branch table entry for the new node. must be done * here, because the next step will destroy the required index char! */ newparentptr = &(r->subtree[c[offs]]); r->subtree[c[offs]] = tree; /* finally fix existing common prefix */ newlen = tree->lenPrefix - offs - 1; if(tree->lenPrefix > sizeof(tree->prefix) && (newlen <= sizeof(tree->prefix))) { /* note: c is a different pointer; the original * pointer is overwritten by memcpy! */ LN_DBGPRINTF(tree->ctx, "splitTree new case one bb, offs %u, lenPrefix %u, newlen %u", offs, tree->lenPrefix, newlen); //LN_DBGPRINTF(tree->ctx, "splitTree new case one bb, *(c+offs): '%s'", c); memcpy(tree->prefix.data, c+offs+1, newlen); free(c); } else { LN_DBGPRINTF(tree->ctx, "splitTree new case two bb, offs=%u, newlen %u", offs, newlen); memmove(c, c+offs+1, newlen); } tree->lenPrefix = tree->lenPrefix - offs - 1; if(tree->parentptr == 0) tree->ctx->ptree = r; /* root does not have a parent! */ else *(tree->parentptr) = r; tree->parentptr = newparentptr; done: return r; } struct ln_ptree * ln_buildPTree(struct ln_ptree *tree, es_str_t *str, size_t offs) { struct ln_ptree *r; unsigned char *c; unsigned char *cpfix; size_t i; unsigned short ipfix; assert(tree != NULL); LN_DBGPRINTF(tree->ctx, "buildPTree: begin at %p, offs %zu", tree, offs); c = es_getBufAddr(str); /* check if the prefix matches and, if not, at what offset it is different */ ipfix = 0; cpfix = prefixBase(tree); for( i = offs ; (i < es_strlen(str)) && (ipfix < tree->lenPrefix) && (c[i] == cpfix[ipfix]) ; ++i, ++ipfix) { ; /*DO NOTHING - just find end of match */ LN_DBGPRINTF(tree->ctx, "buildPTree: tree %p, i %zu, char '%c'", tree, i, c[i]); } /* if we reach this point, we have processed as much of the common prefix * as we could. The following code now does the proper actions based on * the possible cases. */ if(i == es_strlen(str)) { /* all of our input is consumed, no more recursion */ if(ipfix == tree->lenPrefix) { LN_DBGPRINTF(tree->ctx, "case 1.1"); /* exact match, we are done! */ r = tree; } else { LN_DBGPRINTF(tree->ctx, "case 1.2"); /* we need to split the node at the current position */ r = splitTree(tree, ipfix); } } else if(ipfix < tree->lenPrefix) { LN_DBGPRINTF(tree->ctx, "case 2, i=%zu, ipfix=%u", i, ipfix); /* we need to split the node at the current position */ if((r = splitTree(tree, ipfix)) == NULL) goto done; /* fail */ LN_DBGPRINTF(tree->ctx, "pre addPTree: i %zu", i); if((r = ln_addPTree(r, str, i)) == NULL) goto done; //r = ln_buildPTree(r, str, i + 1); } else { /* we could consume the current common prefix, but now need * to traverse the rest of the tree based on the next char. */ if(tree->subtree[c[i]] == NULL) { LN_DBGPRINTF(tree->ctx, "case 3.1"); /* non-match, need new subtree */ r = ln_addPTree(tree, str, i); } else { LN_DBGPRINTF(tree->ctx, "case 3.2"); /* match, follow subtree */ r = ln_buildPTree(tree->subtree[c[i]], str, i + 1); } } //LN_DBGPRINTF(tree->ctx, "---------------------------------------"); //ln_displayPTree(tree, 0); //LN_DBGPRINTF(tree->ctx, "======================================="); done: return r; } int ln_addFDescrToPTree(struct ln_ptree **tree, ln_fieldList_t *node) { int r; ln_fieldList_t *curr; assert(tree != NULL);assert(*tree != NULL); assert(node != NULL); if((node->subtree = ln_newPTree((*tree)->ctx, &node->subtree)) == NULL) { r = -1; goto done; } LN_DBGPRINTF((*tree)->ctx, "got new subtree %p", node->subtree); /* check if we already have this field, if so, merge * TODO: optimized, check logic */ for(curr = (*tree)->froot ; curr != NULL ; curr = curr->next) { if(!es_strcmp(curr->name, node->name) && curr->parser == node->parser && ((curr->raw_data == NULL && node->raw_data == NULL) || (curr->raw_data != NULL && node->raw_data != NULL && !es_strcmp(curr->raw_data, node->raw_data)))) { *tree = curr->subtree; ln_deletePTreeNode(node); r = 0; LN_DBGPRINTF((*tree)->ctx, "merging with tree %p\n", *tree); goto done; } } if((*tree)->froot == NULL) { (*tree)->froot = (*tree)->ftail = node; } else { (*tree)->ftail->next = node; (*tree)->ftail = node; } r = 0; LN_DBGPRINTF((*tree)->ctx, "prev subtree %p", *tree); *tree = node->subtree; LN_DBGPRINTF((*tree)->ctx, "new subtree %p", *tree); done: return r; } void ln_displayPTree(struct ln_ptree *tree, int level) { int i; int nChildLit; int nChildField; es_str_t *str; char *cstr; ln_fieldList_t *node; char indent[2048]; if(level > 1023) level = 1023; memset(indent, ' ', level * 2); indent[level * 2] = '\0'; nChildField = 0; for(node = tree->froot ; node != NULL ; node = node->next ) { ++nChildField; } nChildLit = 0; for(i = 0 ; i < 256 ; ++i) { if(tree->subtree[i] != NULL) { nChildLit++; } } str = es_newStr(sizeof(tree->prefix)); es_addBuf(&str, (char*) prefixBase(tree), tree->lenPrefix); cstr = es_str2cstr(str, NULL); es_deleteStr(str); LN_DBGPRINTF(tree->ctx, "%ssubtree%s %p (prefix: '%s', children: %d literals, %d fields) [visited %u " "backtracked %u terminated %u]", indent, tree->flags.isTerminal ? " TERM" : "", tree, cstr, nChildLit, nChildField, tree->stats.visited, tree->stats.backtracked, tree->stats.terminated); free(cstr); /* display char subtrees */ for(i = 0 ; i < 256 ; ++i) { if(tree->subtree[i] != NULL) { LN_DBGPRINTF(tree->ctx, "%schar %2.2x(%c):", indent, i, i); ln_displayPTree(tree->subtree[i], level + 1); } } /* display field subtrees */ for(node = tree->froot ; node != NULL ; node = node->next ) { cstr = es_str2cstr(node->name, NULL); LN_DBGPRINTF(tree->ctx, "%sfield %s:", indent, cstr); free(cstr); ln_displayPTree(node->subtree, level + 1); } } /* the following is a quick hack, which should be moved to the * string class. */ static inline void dotAddPtr(es_str_t **str, void *p) { char buf[64]; int i; i = snprintf(buf, sizeof(buf), "%p", p); es_addBuf(str, buf, i); } /** * recursive handler for DOT graph generator. */ static void ln_genDotPTreeGraphRec(struct ln_ptree *tree, es_str_t **str) { int i; ln_fieldList_t *node; dotAddPtr(str, tree); es_addBufConstcstr(str, " [label=\""); if(tree->lenPrefix > 0) { es_addChar(str, '\''); es_addBuf(str, (char*) prefixBase(tree), tree->lenPrefix); es_addChar(str, '\''); } es_addBufConstcstr(str, "\""); if(isLeaf(tree)) { es_addBufConstcstr(str, " style=\"bold\""); } es_addBufConstcstr(str, "]\n"); /* display char subtrees */ for(i = 0 ; i < 256 ; ++i) { if(tree->subtree[i] != NULL) { dotAddPtr(str, tree); es_addBufConstcstr(str, " -> "); dotAddPtr(str, tree->subtree[i]); es_addBufConstcstr(str, " [label=\""); es_addChar(str, (char) i); es_addBufConstcstr(str, "\"]\n"); ln_genDotPTreeGraphRec(tree->subtree[i], str); } } /* display field subtrees */ for(node = tree->froot ; node != NULL ; node = node->next ) { dotAddPtr(str, tree); es_addBufConstcstr(str, " -> "); dotAddPtr(str, node->subtree); es_addBufConstcstr(str, " [label=\""); es_addStr(str, node->name); es_addBufConstcstr(str, "\" style=\"dotted\"]\n"); ln_genDotPTreeGraphRec(node->subtree, str); } } void ln_genDotPTreeGraph(struct ln_ptree *tree, es_str_t **str) { es_addBufConstcstr(str, "digraph ptree {\n"); ln_genDotPTreeGraphRec(tree, str); es_addBufConstcstr(str, "}\n"); } /** * add unparsed string to event. */ static int addUnparsedField(const char *str, size_t strLen, int offs, struct json_object *json) { int r = 1; struct json_object *value; char *s = NULL; CHKN(s = strndup(str, strLen)); value = json_object_new_string(s); if (value == NULL) { goto done; } json_object_object_add(json, ORIGINAL_MSG_KEY, value); value = json_object_new_string(s + offs); if (value == NULL) { goto done; } json_object_object_add(json, UNPARSED_DATA_KEY, value); r = 0; done: free(s); return r; } /** * Special parser for iptables-like name/value pairs. * The pull multiple fields. Note that once this parser has been selected, * it is very unlikely to be left, as it is *very* generic. This parser is * required because practice shows that already-structured data like iptables * can otherwise not be processed by liblognorm in a meaningful way. * * @param[in] tree current tree to process * @param[in] str string to be matched against (the to-be-normalized data) * @param[in] strLen length of str * @param[in/out] offs start position in input data, on exit first unparsed position * @param[in/out] event handle to event that is being created during normalization * * @return 0 if parser was successfully, something else on error */ static int ln_iptablesParser(struct ln_ptree *tree, const char *str, size_t strLen, size_t *offs, struct json_object *json) { int r; size_t o = *offs; es_str_t *fname; es_str_t *fval; const char *pstr; const char *end; struct json_object *value; LN_DBGPRINTF(tree->ctx, "%zu enter iptables parser, len %zu", *offs, strLen); if(o == strLen) { r = -1; /* can not be, we have no n/v pairs! */ goto done; } end = str + strLen; pstr = str + o; while(pstr < end) { while(pstr < end && isspace(*pstr)) ++pstr; CHKN(fname = es_newStr(16)); while(pstr < end && !isspace(*pstr) && *pstr != '=') { es_addChar(&fname, *pstr); ++pstr; } if(pstr < end && *pstr == '=') { CHKN(fval = es_newStr(16)); ++pstr; /* error on space */ while(pstr < end && !isspace(*pstr)) { es_addChar(&fval, *pstr); ++pstr; } } else { CHKN(fval = es_newStrFromCStr("[*PRESENT*]", sizeof("[*PRESENT*]")-1)); } char *cn, *cv; CHKN(cn = ln_es_str2cstr(&fname)); CHKN(cv = ln_es_str2cstr(&fval)); if (tree->ctx->debug) { LN_DBGPRINTF(tree->ctx, "iptables parser extracts %s=%s", cn, cv); } CHKN(value = json_object_new_string(cv)); json_object_object_add(json, cn, value); es_deleteStr(fval); es_deleteStr(fname); } r = 0; *offs = strLen; done: LN_DBGPRINTF(tree->ctx, "%zu iptables parser returns %d", *offs, r); return r; } /** * Recursive step of the normalizer. It walks the parse tree and calls itself * recursively when this is appropriate. It also implements backtracking in * those (hopefully rare) cases where it is required. * * @param[in] tree current tree to process * @param[in] string string to be matched against (the to-be-normalized data) * @param[in] strLen length of the to-be-matched string * @param[in] offs start position in input data * @param[in/out] json ... that is being created during normalization * @param[out] endNode if a match was found, this is the matching node (undefined otherwise) * * @return number of characters left unparsed by following the subtree, negative if * the to-be-parsed message is shorter than the rule sample by this number of * characters. */ static int ln_v1_normalizeRec(struct ln_ptree *tree, const char *str, size_t strLen, size_t offs, struct json_object *json, struct ln_ptree **endNode) { int r; int localR; size_t i; int left; ln_fieldList_t *node; ln_fieldList_t *restMotifNode = NULL; char *cstr; const char *c; unsigned char *cpfix; unsigned ipfix; size_t parsed; char *namestr; struct json_object *value; ++tree->stats.visited; if(offs >= strLen) { *endNode = tree; r = -tree->lenPrefix; goto done; } LN_DBGPRINTF(tree->ctx, "%zu: enter parser, tree %p", offs, tree); c = str; cpfix = prefixBase(tree); node = tree->froot; r = strLen - offs; /* first we need to check if the common prefix matches (and consume input data while we do) */ ipfix = 0; while(offs < strLen && ipfix < tree->lenPrefix) { LN_DBGPRINTF(tree->ctx, "%zu: prefix compare '%c', '%c'", offs, c[offs], cpfix[ipfix]); if(c[offs] != cpfix[ipfix]) { r -= ipfix; goto done; } ++offs, ++ipfix; } if(ipfix != tree->lenPrefix) { /* incomplete prefix match --> to-be-normalized string too short */ r = ipfix - tree->lenPrefix; goto done; } r -= ipfix; LN_DBGPRINTF(tree->ctx, "%zu: prefix compare succeeded, still valid", offs); /* now try the parsers */ while(node != NULL) { if(tree->ctx->debug) { cstr = es_str2cstr(node->name, NULL); LN_DBGPRINTF(tree->ctx, "%zu:trying parser for field '%s': %p", offs, cstr, node->parser); free(cstr); } i = offs; if(node->isIPTables) { localR = ln_iptablesParser(tree, str, strLen, &i, json); LN_DBGPRINTF(tree->ctx, "%zu iptables parser return, i=%zu", offs, i); if(localR == 0) { /* potential hit, need to verify */ LN_DBGPRINTF(tree->ctx, "potential hit, trying subtree"); left = ln_v1_normalizeRec(node->subtree, str, strLen, i, json, endNode); if(left == 0 && (*endNode)->flags.isTerminal) { LN_DBGPRINTF(tree->ctx, "%zu: parser matches at %zu", offs, i); r = 0; goto done; } LN_DBGPRINTF(tree->ctx, "%zu nonmatch, backtracking required, left=%d", offs, left); ++tree->stats.backtracked; if(left < r) r = left; } } else if(node->parser == ln_parseRest) { /* This is a quick and dirty adjustment to handle "rest" more intelligently. * It's just a tactical fix: in the longer term, we'll handle the whole * situation differently. However, it makes sense to fix this now, as this * solves some real-world problems immediately. -- rgerhards, 2015-04-15 */ restMotifNode = node; } else { value = NULL; localR = node->parser(str, strLen, &i, node, &parsed, &value); LN_DBGPRINTF(tree->ctx, "parser returns %d, parsed %zu", localR, parsed); if(localR == 0) { /* potential hit, need to verify */ LN_DBGPRINTF(tree->ctx, "%zu: potential hit, trying subtree %p", offs, node->subtree); left = ln_v1_normalizeRec(node->subtree, str, strLen, i + parsed, json, endNode); LN_DBGPRINTF(tree->ctx, "%zu: subtree returns %d", offs, r); if(left == 0 && (*endNode)->flags.isTerminal) { LN_DBGPRINTF(tree->ctx, "%zu: parser matches at %zu", offs, i); if(es_strbufcmp(node->name, (unsigned char*)"-", 1)) { /* Store the value here; create json if not already created */ if (value == NULL) { CHKN(cstr = strndup(str + i, parsed)); value = json_object_new_string(cstr); free(cstr); } if (value == NULL) { LN_DBGPRINTF(tree->ctx, "unable to create json"); goto done; } namestr = ln_es_str2cstr(&node->name); json_object_object_add(json, namestr, value); } else { if (value != NULL) { /* Free the unneeded value */ json_object_put(value); } } r = 0; goto done; } LN_DBGPRINTF(tree->ctx, "%zu nonmatch, backtracking required, left=%d", offs, left); if (value != NULL) { /* Free the value if it was created */ json_object_put(value); } if(left > 0 && left < r) r = left; LN_DBGPRINTF(tree->ctx, "%zu nonmatch, backtracking required, left=%d, r now %d", offs, left, r); ++tree->stats.backtracked; } } node = node->next; } if(offs == strLen) { *endNode = tree; r = 0; goto done; } if(offs < strLen) { unsigned char cc = str[offs]; LN_DBGPRINTF(tree->ctx, "%zu no field, trying subtree char '%c': %p", offs, cc, tree->subtree[cc]); } else { LN_DBGPRINTF(tree->ctx, "%zu no field, offset already beyond end", offs); } /* now let's see if we have a literal */ if(tree->subtree[(unsigned char)str[offs]] != NULL) { left = ln_v1_normalizeRec(tree->subtree[(unsigned char)str[offs]], str, strLen, offs + 1, json, endNode); LN_DBGPRINTF(tree->ctx, "%zu got left %d, r %d", offs, left, r); if(left < r) r = left; LN_DBGPRINTF(tree->ctx, "%zu got return %d", offs, r); } if(r == 0 && (*endNode)->flags.isTerminal) goto done; /* and finally give "rest" a try if it was present. Note that we MUST do this after * literal evaluation, otherwise "rest" can never be overridden by other rules. */ if(restMotifNode != NULL) { LN_DBGPRINTF(tree->ctx, "rule has rest motif, forcing match via it"); value = NULL; restMotifNode->parser(str, strLen, &i, restMotifNode, &parsed, &value); # ifndef NDEBUG left = /* we only need this for the assert below */ # endif ln_v1_normalizeRec(restMotifNode->subtree, str, strLen, i + parsed, json, endNode); assert(left == 0); /* with rest, we have this invariant */ assert((*endNode)->flags.isTerminal); /* this one also */ LN_DBGPRINTF(tree->ctx, "%zu: parser matches at %zu", offs, i); if(es_strbufcmp(restMotifNode->name, (unsigned char*)"-", 1)) { /* Store the value here; create json if not already created */ if (value == NULL) { CHKN(cstr = strndup(str + i, parsed)); value = json_object_new_string(cstr); free(cstr); } if (value == NULL) { LN_DBGPRINTF(tree->ctx, "unable to create json"); goto done; } namestr = ln_es_str2cstr(&restMotifNode->name); json_object_object_add(json, namestr, value); } else { if (value != NULL) { /* Free the unneeded value */ json_object_put(value); } } r = 0; goto done; } done: LN_DBGPRINTF(tree->ctx, "%zu returns %d", offs, r); if(r == 0 && *endNode == tree) ++tree->stats.terminated; return r; } int ln_v1_normalize(ln_ctx ctx, const char *str, size_t strLen, struct json_object **json_p) { int r; int left; struct ln_ptree *endNode = NULL; if(*json_p == NULL) { CHKN(*json_p = json_object_new_object()); } left = ln_v1_normalizeRec(ctx->ptree, str, strLen, 0, *json_p, &endNode); if(ctx->debug) { if(left == 0) { LN_DBGPRINTF(ctx, "final result for normalizer: left %d, endNode %p, " "isTerminal %d, tagbucket %p", left, endNode, endNode->flags.isTerminal, endNode->tags); } else { LN_DBGPRINTF(ctx, "final result for normalizer: left %d, endNode %p", left, endNode); } } if(left != 0 || !endNode->flags.isTerminal) { /* we could not successfully parse, some unparsed items left */ if(left < 0) { addUnparsedField(str, strLen, strLen, *json_p); } else { addUnparsedField(str, strLen, strLen - left, *json_p); } } else { /* success, finalize event */ if(endNode->tags != NULL) { /* add tags to an event */ json_object_get(endNode->tags); json_object_object_add(*json_p, "event.tags", endNode->tags); CHKR(ln_annotate(ctx, *json_p, endNode->tags)); } } r = 0; done: return r; } /** * Gather and output pdag statistics for the full pdag (ctx) * including all disconnected components (type defs). * * Data is sent to given file ptr. */ void ln_fullPTreeStats(ln_ctx ctx, FILE __attribute__((unused)) *const fp, const int __attribute__((unused)) extendedStats) { ln_displayPTree(ctx->ptree, 0); } liblognorm-2.0.8/src/v1_ptree.h000066400000000000000000000163571511425433100163650ustar00rootroot00000000000000/** * @file ptree.h * @brief The parse tree object. * @class ln_ptree ptree.h *//* * Copyright 2013 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is meant to be included by applications using liblognorm. * For lognorm library files themselves, include "lognorm.h". * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef LIBLOGNORM_PTREE_H_INCLUDED #define LIBLOGNORM_PTREE_H_INCLUDED #include #include #define ORIGINAL_MSG_KEY "originalmsg" #define UNPARSED_DATA_KEY "unparsed-data" typedef struct ln_ptree ln_ptree; /**< the parse tree object */ typedef struct ln_fieldList_s ln_fieldList_t; /** * List of supported fields inside parse tree. * This list holds all fields and their description. While normalizing, * fields are tried in the order of this list. So the enqueue order * dictates precedence during parsing. * * value list. This is a single-linked list. In a later stage, we should * optimize it so that frequently used fields are moved "up" towards * the root of the list. In any case, we do NOT expect this list to * be long, as the parser should already have gotten quite specific when * we hit a fieldconst . */ struct ln_fieldList_s { es_str_t *name; /**< field name */ es_str_t *data; /**< extra data to be passed to parser */ es_str_t *raw_data; /**< extra untouched (unescaping is not done) data available to be used by parser */ void *parser_data; /** opaque data that the field-parser understands */ void (*parser_data_destructor)(void **); /** destroy opaque data that field-parser understands */ int (*parser)(const char*, size_t, size_t*, const ln_fieldList_t *, size_t*, struct json_object **); /**< parser to use */ ln_ptree *subtree; /**< subtree to follow if parser succeeded */ ln_fieldList_t *next; /**< list housekeeping, next node (or NULL) */ unsigned char isIPTables; /**< special parser: iptables! */ }; /* parse tree object */ struct ln_ptree { ln_ctx ctx; /**< our context */ ln_ptree **parentptr; /**< pointer to *us* *inside* the parent BUT this is NOT a pointer to the parent! */ ln_fieldList_t *froot; /**< root of field list */ ln_fieldList_t *ftail; /**< tail of field list */ struct { unsigned isTerminal:1; /**< designates this node a terminal sequence? */ } flags; struct json_object *tags; /* tags to assign to events of this type */ /* the representation below requires a lof of memory but is * very fast. As an alternate approach, we can use a hash table * where we ignore control characters. That should work quite well. * But we do not do this in the initial step. */ ln_ptree *subtree[256]; unsigned short lenPrefix; /**< length of common prefix, 0->none */ union { unsigned char *ptr; /**< use if data element is too large */ unsigned char data[16]; /**< fast lookup for small string */ } prefix; /**< a common prefix string for all of this node */ struct { unsigned visited; unsigned backtracked; /**< incremented when backtracking was initiated */ unsigned terminated; } stats; /**< usage statistics */ }; /* Methods */ /** * Allocates and initializes a new parse tree node. * @memberof ln_ptree * * @param[in] ctx current library context. This MUST match the * context of the parent. * @param[in] parent pointer to the new node inside the parent * * @return pointer to new node or NULL on error */ struct ln_ptree* ln_newPTree(ln_ctx ctx, struct ln_ptree** parent); /** * Free a parse tree and destruct all members. * @memberof ln_ptree * * @param[in] tree pointer to ptree to free */ void ln_deletePTree(struct ln_ptree *tree); /** * Free a parse tree node and destruct all members. * @memberof ln_ptree * * @param[in] node pointer to free */ void ln_deletePTreeNode(ln_fieldList_t *node); /** * Add a field description to the a tree. * The field description will be added as last field. Fields are * parsed in the order they have been added, so be sure to care * about the order if that matters. * @memberof ln_ptree * * @param[in] tree pointer to ptree to modify * @param[in] fielddescr a fully populated (and initialized) * field description node * @returns 0 on success, something else otherwise */ int ln_addFDescrToPTree(struct ln_ptree **tree, ln_fieldList_t *node); /** * Add a literal to a ptree. * Creates new tree nodes as necessary. * @memberof ln_ptree * * @param[in] tree root of tree where to add * @param[in] str literal (string) to add * @param[in] offs offset of where in literal adding should start * * @return NULL on error, otherwise pointer to deepest tree added */ struct ln_ptree* ln_addPTree(struct ln_ptree *tree, es_str_t *str, size_t offs); /** * Display the content of a ptree (debug function). * This is a debug aid that spits out a textual representation * of the provided ptree via multiple calls of the debug callback. * * @param tree ptree to display * @param level recursion level, must be set to 0 on initial call */ void ln_displayPTree(struct ln_ptree *tree, int level); /** * Generate a DOT graph. * Well, actually it does not generate the graph itself, but a * control file that is suitable for the GNU DOT tool. Such a file * can be very useful to understand complex sample databases * (not to mention that it is probably fun for those creating * samples). * The dot commands are appended to the provided string. * * @param[in] tree ptree to display * @param[out] str string which receives the DOT commands. */ void ln_genDotPTreeGraph(struct ln_ptree *tree, es_str_t **str); /** * Build a ptree based on the provided string, but only if necessary. * The passed-in tree is searched and traversed for str. If a node exactly * matching str is found, that node is returned. If no exact match is found, * a new node is added. Existing nodes may be split, if a so-far common * prefix needs to be split in order to add the new node. * * @param[in] tree root of the current tree * @param[in] str string to be added * @param[in] offs offset into str where match needs to start * (this is required for recursive calls to handle * common prefixes) * @return NULL on error, otherwise the ptree leaf that * corresponds to the parameters passed. */ struct ln_ptree * ln_buildPTree(struct ln_ptree *tree, es_str_t *str, size_t offs); /* internal helper for displaying stats */ void ln_fullPTreeStats(ln_ctx ctx, FILE *const fp, const int extendedStats); #endif /* #ifndef LOGNORM_PTREE_H_INCLUDED */ liblognorm-2.0.8/src/v1_samp.c000066400000000000000000000557441511425433100162040ustar00rootroot00000000000000/* samp.c -- code for ln_v1_samp objects. * * Copyright 2010-2015 by Rainer Gerhards and Adiscon GmbH. * * Modified by Pavel Levshin (pavel@levshin.spb.ru) in 2013 * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #include "config.h" #include #include #include #include #include #include #define LOGNORM_V1_SUBSYSTEM /* indicate we are old cruft */ #include "v1_liblognorm.h" #include "internal.h" #include "lognorm.h" #include "samp.h" #include "v1_ptree.h" #include "v1_samp.h" #include "v1_parser.h" /** * Construct a sample object. */ struct ln_v1_samp* ln_v1_sampCreate(ln_ctx __attribute__((unused)) ctx) { struct ln_v1_samp* samp; if((samp = calloc(1, sizeof(struct ln_v1_samp))) == NULL) goto done; /* place specific init code here (none at this time) */ done: return samp; } void ln_v1_sampFree(ln_ctx __attribute__((unused)) ctx, struct ln_v1_samp *samp) { free(samp); } /** * Extract a field description from a sample. * The field description is added to the tail of the current * subtree's field list. The parse buffer must be position on the * leading '%' that starts a field definition. It is a program error * if this condition is not met. * * Note that we break up the object model and access ptree members * directly. Let's consider us a friend of ptree. This is necessary * to optimize the structure for a high-speed parsing process. * * @param[in] str a temporary work string. This is passed in to save the * creation overhead * @returns 0 on success, something else otherwise */ static int addFieldDescr(ln_ctx ctx, struct ln_ptree **subtree, es_str_t *rule, es_size_t *bufOffs, es_str_t **str) { int r; ln_fieldList_t *node = ln_v1_parseFieldDescr(ctx, rule, bufOffs, str, &r); assert(subtree != NULL); if (node != NULL) CHKR(ln_addFDescrToPTree(subtree, node)); done: return r; } ln_fieldList_t* ln_v1_parseFieldDescr(ln_ctx ctx, es_str_t *rule, es_size_t *bufOffs, es_str_t **str, int* ret) { int r = 0; ln_fieldList_t *node; es_size_t i = *bufOffs; char *cstr; /* for debug mode strings */ unsigned char *buf; es_size_t lenBuf; void* (*constructor_fn)(ln_fieldList_t *, ln_ctx) = NULL; buf = es_getBufAddr(rule); lenBuf = es_strlen(rule); assert(buf[i] == '%'); ++i; /* "eat" ':' */ CHKN(node = calloc(1, sizeof(ln_fieldList_t))); node->subtree = NULL; node->next = NULL; node->data = NULL; node->raw_data = NULL; node->parser_data = NULL; node->parser_data_destructor = NULL; CHKN(node->name = es_newStr(16)); /* skip leading whitespace in field name */ while(i < lenBuf && isspace(buf[i])) ++i; while(i < lenBuf && buf[i] != ':') { CHKR(es_addChar(&node->name, buf[i++])); } if(es_strlen(node->name) == 0) { FAIL(LN_INVLDFDESCR); } if(ctx->debug) { cstr = es_str2cstr(node->name, NULL); ln_dbgprintf(ctx, "parsed field: '%s'", cstr); free(cstr); } if(buf[i] != ':') { /* may be valid later if we have a loaded CEE dictionary * and the name is present inside it. */ FAIL(LN_INVLDFDESCR); } ++i; /* skip ':' */ /* parse and process type (trailing whitespace must be trimmed) */ es_emptyStr(*str); size_t j = i; /* scan for terminator */ while(j < lenBuf && buf[j] != ':' && buf[j] != '%') ++j; /* now trim trailing space backwards */ size_t next = j; --j; while(j >= i && isspace(buf[j])) --j; /* now copy */ while(i <= j) { CHKR(es_addChar(str, buf[i++])); } /* finally move i to consumed position */ i = next; if(i == lenBuf) { FAIL(LN_INVLDFDESCR); } node->isIPTables = 0; /* first assume no special parser is used */ if(!es_strconstcmp(*str, "date-rfc3164")) { node->parser = ln_parseRFC3164Date; } else if(!es_strconstcmp(*str, "date-rfc5424")) { node->parser = ln_parseRFC5424Date; } else if(!es_strconstcmp(*str, "number")) { node->parser = ln_parseNumber; } else if(!es_strconstcmp(*str, "float")) { node->parser = ln_parseFloat; } else if(!es_strconstcmp(*str, "hexnumber")) { node->parser = ln_parseHexNumber; } else if(!es_strconstcmp(*str, "kernel-timestamp")) { node->parser = ln_parseKernelTimestamp; } else if(!es_strconstcmp(*str, "whitespace")) { node->parser = ln_parseWhitespace; } else if(!es_strconstcmp(*str, "ipv4")) { node->parser = ln_parseIPv4; } else if(!es_strconstcmp(*str, "ipv6")) { node->parser = ln_parseIPv6; } else if(!es_strconstcmp(*str, "word")) { node->parser = ln_parseWord; } else if(!es_strconstcmp(*str, "alpha")) { node->parser = ln_parseAlpha; } else if(!es_strconstcmp(*str, "rest")) { node->parser = ln_parseRest; } else if(!es_strconstcmp(*str, "op-quoted-string")) { node->parser = ln_parseOpQuotedString; } else if(!es_strconstcmp(*str, "quoted-string")) { node->parser = ln_parseQuotedString; } else if(!es_strconstcmp(*str, "date-iso")) { node->parser = ln_parseISODate; } else if(!es_strconstcmp(*str, "time-24hr")) { node->parser = ln_parseTime24hr; } else if(!es_strconstcmp(*str, "time-12hr")) { node->parser = ln_parseTime12hr; } else if(!es_strconstcmp(*str, "duration")) { node->parser = ln_parseDuration; } else if(!es_strconstcmp(*str, "cisco-interface-spec")) { node->parser = ln_parseCiscoInterfaceSpec; } else if(!es_strconstcmp(*str, "json")) { node->parser = ln_parseJSON; } else if(!es_strconstcmp(*str, "cee-syslog")) { node->parser = ln_parseCEESyslog; } else if(!es_strconstcmp(*str, "mac48")) { node->parser = ln_parseMAC48; } else if(!es_strconstcmp(*str, "name-value-list")) { node->parser = ln_parseNameValue; } else if(!es_strconstcmp(*str, "cef")) { node->parser = ln_parseCEF; } else if(!es_strconstcmp(*str, "checkpoint-lea")) { node->parser = ln_parseCheckpointLEA; } else if(!es_strconstcmp(*str, "v2-iptables")) { node->parser = ln_parsev2IPTables; } else if(!es_strconstcmp(*str, "iptables")) { node->parser = NULL; node->isIPTables = 1; } else if(!es_strconstcmp(*str, "string-to")) { /* TODO: check extra data!!!! (very important) */ node->parser = ln_parseStringTo; } else if(!es_strconstcmp(*str, "char-to")) { /* TODO: check extra data!!!! (very important) */ node->parser = ln_parseCharTo; } else if(!es_strconstcmp(*str, "char-sep")) { /* TODO: check extra data!!!! (very important) */ node->parser = ln_parseCharSeparated; } else if(!es_strconstcmp(*str, "tokenized")) { node->parser = ln_parseTokenized; constructor_fn = tokenized_parser_data_constructor; node->parser_data_destructor = tokenized_parser_data_destructor; } #ifdef FEATURE_REGEXP else if(!es_strconstcmp(*str, "regex")) { node->parser = ln_parseRegex; constructor_fn = regex_parser_data_constructor; node->parser_data_destructor = regex_parser_data_destructor; } #endif else if (!es_strconstcmp(*str, "recursive")) { node->parser = ln_parseRecursive; constructor_fn = recursive_parser_data_constructor; node->parser_data_destructor = recursive_parser_data_destructor; } else if (!es_strconstcmp(*str, "descent")) { node->parser = ln_parseRecursive; constructor_fn = descent_parser_data_constructor; node->parser_data_destructor = recursive_parser_data_destructor; } else if (!es_strconstcmp(*str, "interpret")) { node->parser = ln_parseInterpret; constructor_fn = interpret_parser_data_constructor; node->parser_data_destructor = interpret_parser_data_destructor; } else if (!es_strconstcmp(*str, "suffixed")) { node->parser = ln_parseSuffixed; constructor_fn = suffixed_parser_data_constructor; node->parser_data_destructor = suffixed_parser_data_destructor; } else if (!es_strconstcmp(*str, "named_suffixed")) { node->parser = ln_parseSuffixed; constructor_fn = named_suffixed_parser_data_constructor; node->parser_data_destructor = suffixed_parser_data_destructor; } else { cstr = es_str2cstr(*str, NULL); ln_errprintf(ctx, 0, "invalid field type '%s'", cstr); free(cstr); FAIL(LN_INVLDFDESCR); } if(buf[i] == '%') { i++; } else { /* parse extra data */ CHKN(node->data = es_newStr(8)); i++; while(i < lenBuf) { if(buf[i] == '%') { ++i; break; /* end of field */ } CHKR(es_addChar(&node->data, buf[i++])); } node->raw_data = es_strdup(node->data); es_unescapeStr(node->data); if(ctx->debug) { cstr = es_str2cstr(node->data, NULL); ln_dbgprintf(ctx, "parsed extra data: '%s'", cstr); free(cstr); } } if (constructor_fn) node->parser_data = constructor_fn(node, ctx); *bufOffs = i; done: if (r != 0) { if (node->name != NULL) es_deleteStr(node->name); free(node); node = NULL; } *ret = r; return node; } /** * Parse a Literal string out of the template and add it to the tree. * @param[in] ctx the context * @param[in/out] subtree on entry, current subtree, on exist newest * deepest subtree * @param[in] rule string with current rule * @param[in/out] bufOffs parse pointer, up to which offset is parsed * (is updated so that it points to first char after consumed * string on exit). * @param[out] str literal extracted (is empty, when no litral could be found) * @return 0 on success, something else otherwise */ static int parseLiteral(ln_ctx ctx, struct ln_ptree **subtree, es_str_t *rule, es_size_t *bufOffs, es_str_t **str) { int r = 0; es_size_t i = *bufOffs; unsigned char *buf; es_size_t lenBuf; es_emptyStr(*str); buf = es_getBufAddr(rule); lenBuf = es_strlen(rule); /* extract maximum length literal */ while(i < lenBuf) { if(buf[i] == '%') { if(i+1 < lenBuf && buf[i+1] != '%') { break; /* field start is end of literal */ } if (++i == lenBuf) break; } CHKR(es_addChar(str, buf[i])); ++i; } es_unescapeStr(*str); if(ctx->debug) { char *cstr = es_str2cstr(*str, NULL); ln_dbgprintf(ctx, "parsed literal: '%s'", cstr); free(cstr); } *subtree = ln_buildPTree(*subtree, *str, 0); *bufOffs = i; r = 0; done: return r; } /* Implementation note: * We read in the sample, and split it into chunks of literal text and * fields. Each literal text is added as whole to the tree, as is each * field individually. To do so, we keep track of our current subtree * root, which changes whenever a new part of the tree is build. It is * set to the then-lowest part of the tree, where the next step sample * data is to be added. * * This function processes the whole string or returns an error. * * format: literal1%field:type:extra-data%literal2 * * @returns the new subtree root (or NULL in case of error) */ static int addSampToTree(ln_ctx ctx, es_str_t *rule, struct json_object *tagBucket) { int r = -1; struct ln_ptree* subtree; es_str_t *str = NULL; es_size_t i; subtree = ctx->ptree; CHKN(str = es_newStr(256)); i = 0; while(i < es_strlen(rule)) { ln_dbgprintf(ctx, "addSampToTree %d of %d", i, es_strlen(rule)); CHKR(parseLiteral(ctx, &subtree, rule, &i, &str)); /* After the literal there can be field only*/ if (i < es_strlen(rule)) { CHKR(addFieldDescr(ctx, &subtree, rule, &i, &str)); if (i == es_strlen(rule)) { /* finish the tree with empty literal to avoid false merging*/ CHKR(parseLiteral(ctx, &subtree, rule, &i, &str)); } } } ln_dbgprintf(ctx, "end addSampToTree %d of %d", i, es_strlen(rule)); /* we are at the end of rule processing, so this node is a terminal */ subtree->flags.isTerminal = 1; subtree->tags = tagBucket; done: if(str != NULL) es_deleteStr(str); return r; } /** * get the initial word of a rule line that tells us the type of the * line. * @param[in] buf line buffer * @param[in] len length of buffer * @param[out] offs offset after "=" * @param[out] str string with "linetype-word" (newly created) * @returns 0 on success, something else otherwise */ static int getLineType(const char *buf, es_size_t lenBuf, es_size_t *offs, es_str_t **str) { int r = -1; es_size_t i; *str = es_newStr(16); for(i = 0 ; i < lenBuf && buf[i] != '=' ; ++i) { CHKR(es_addChar(str, buf[i])); } if(i < lenBuf) ++i; /* skip over '=' */ *offs = i; done: return r; } /** * Get a new common prefix from the config file. That is actually everything from * the current offset to the end of line. * * @param[in] buf line buffer * @param[in] len length of buffer * @param[in] offs offset after "=" * @param[in/out] str string to store common offset. If NULL, it is created, * otherwise it is emptied. * @returns 0 on success, something else otherwise */ static int getPrefix(const char *buf, es_size_t lenBuf, es_size_t offs, es_str_t **str) { int r; if(*str == NULL) { CHKN(*str = es_newStr(lenBuf - offs)); } else { es_emptyStr(*str); } r = es_addBuf(str, (char*)buf + offs, lenBuf - offs); done: return r; } /** * Extend the common prefix. This means that the line is concatenated * to the prefix. This is useful if the same rulebase is to be used with * different prefixes (well, not strictly necessary, but probably useful). * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in] offs offset to-be-added text starts * @returns 0 on success, something else otherwise */ static int extendPrefix(ln_ctx ctx, const char *buf, es_size_t lenBuf, es_size_t offs) { return es_addBuf(&ctx->rulePrefix, (char*)buf+offs, lenBuf - offs); } /** * Add a tag to the tag bucket. Helper to processTags. * @param[in] ctx current context * @param[in] tagname string with tag name * @param[out] tagBucket tagbucket to which new tags shall be added * the tagbucket is created if it is NULL * @returns 0 on success, something else otherwise */ static int addTagStrToBucket(ln_ctx ctx, es_str_t *tagname, struct json_object **tagBucket) { int r = -1; char *cstr; struct json_object *tag; if(*tagBucket == NULL) { CHKN(*tagBucket = json_object_new_array()); } cstr = es_str2cstr(tagname, NULL); ln_dbgprintf(ctx, "tag found: '%s'", cstr); CHKN(tag = json_object_new_string(cstr)); json_object_array_add(*tagBucket, tag); free(cstr); r = 0; done: return r; } /** * Extract the tags and create a tag bucket out of them * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in,out] poffs offset where tags start, on exit and success * offset after tag part (excluding ':') * @param[out] tagBucket tagbucket to which new tags shall be added * the tagbucket is created if it is NULL * @returns 0 on success, something else otherwise */ static int processTags(ln_ctx ctx, const char *buf, es_size_t lenBuf, es_size_t *poffs, struct json_object **tagBucket) { int r = -1; es_str_t *str = NULL; es_size_t i; assert(poffs != NULL); i = *poffs; while(i < lenBuf && buf[i] != ':') { if(buf[i] == ',') { /* end of this tag */ CHKR(addTagStrToBucket(ctx, str, tagBucket)); es_deleteStr(str); str = NULL; } else { if(str == NULL) { CHKN(str = es_newStr(32)); } CHKR(es_addChar(&str, buf[i])); } ++i; } if(buf[i] != ':') goto done; ++i; /* skip ':' */ if(str != NULL) { CHKR(addTagStrToBucket(ctx, str, tagBucket)); es_deleteStr(str); } *poffs = i; r = 0; done: return r; } /** * Process a new rule and add it to tree. * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in] offs offset where rule starts * @returns 0 on success, something else otherwise */ static int processRule(ln_ctx ctx, const char *buf, es_size_t lenBuf, es_size_t offs) { int r = -1; es_str_t *str; struct json_object *tagBucket = NULL; ln_dbgprintf(ctx, "sample line to add: '%s'\n", buf+offs); CHKR(processTags(ctx, buf, lenBuf, &offs, &tagBucket)); if(offs == lenBuf) { ln_dbgprintf(ctx, "error, actual message sample part is missing"); // TODO: provide some error indicator to app? We definitely must do (a callback?) goto done; } if(ctx->rulePrefix == NULL) { CHKN(str = es_newStr(lenBuf)); } else { CHKN(str = es_strdup(ctx->rulePrefix)); } CHKR(es_addBuf(&str, (char*)buf + offs, lenBuf - offs)); addSampToTree(ctx, str, tagBucket); es_deleteStr(str); r = 0; done: return r; } /** * Obtain a field name from a rule base line. * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in/out] offs on entry: offset where tag starts, * on exit: updated offset AFTER TAG and (':') * @param [out] strTag obtained tag, if successful * @returns 0 on success, something else otherwise */ static int getFieldName(ln_ctx __attribute__((unused)) ctx, const char *buf, es_size_t lenBuf, es_size_t *offs, es_str_t **strTag) { int r = -1; es_size_t i; i = *offs; while(i < lenBuf && (isalnum(buf[i]) || buf[i] == '_' || buf[i] == '.')) { if(*strTag == NULL) { CHKN(*strTag = es_newStr(32)); } CHKR(es_addChar(strTag, buf[i])); ++i; } *offs = i; r = 0; done: return r; } /** * Skip over whitespace. * Skips any whitespace present at the offset. * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in/out] offs on entry: offset first unprocessed position */ static void skipWhitespace(ln_ctx __attribute__((unused)) ctx, const char *buf, es_size_t lenBuf, es_size_t *offs) { while(*offs < lenBuf && isspace(buf[*offs])) { (*offs)++; } } /** * Obtain an annotation (field) operation. * This usually is a plus or minus sign followed by a field name * followed (if plus) by an equal sign and the field value. On entry, * offs must be positioned on the first unprocessed field (after ':' for * the initial field!). Extra whitespace is detected and, if present, * skipped. The obtained operation is added to the annotation set provided. * Note that extracted string objects are passed to the annotation; thus it * is vital NOT to free them (most importantly, this is *not* a memory leak). * * @param[in] ctx current context * @param[in] annot active annotation set to which the operation is to be added * @param[in] buf line buffer * @param[in] len length of buffer * @param[in/out] offs on entry: offset where tag starts, * on exit: updated offset AFTER TAG and (':') * @param [out] strTag obtained tag, if successful * @returns 0 on success, something else otherwise */ static int getAnnotationOp(ln_ctx ctx, ln_annot *annot, const char *buf, es_size_t lenBuf, es_size_t *offs) { int r = -1; es_size_t i; es_str_t *fieldName = NULL; es_str_t *fieldVal = NULL; ln_annot_opcode opc; i = *offs; skipWhitespace(ctx, buf, lenBuf, &i); if(i == lenBuf) { r = 0; goto done; /* nothing left to process (no error!) */ } if(buf[i] == '+') { opc = ln_annot_ADD; } else if(buf[i] == '-') { ln_dbgprintf(ctx, "annotate op '-' not yet implemented - failing"); goto fail; } else { ln_dbgprintf(ctx, "invalid annotate opcode '%c' - failing" , buf[i]); goto fail; } i++; if(i == lenBuf) goto fail; /* nothing left to process */ CHKR(getFieldName(ctx, buf, lenBuf, &i, &fieldName)); if(i == lenBuf) goto fail; /* nothing left to process */ if(buf[i] != '=') goto fail; /* format error */ i++; skipWhitespace(ctx, buf, lenBuf, &i); if(buf[i] != '"') goto fail; /* format error */ ++i; while(i < lenBuf && buf[i] != '"') { if(fieldVal == NULL) { CHKN(fieldVal = es_newStr(32)); } CHKR(es_addChar(&fieldVal, buf[i])); ++i; } *offs = (i == lenBuf) ? i : i+1; CHKR(ln_addAnnotOp(annot, opc, fieldName, fieldVal)); r = 0; done: return r; fail: return -1; } /** * Process a new annotation and add it to the annotation set. * * @param[in] ctx current context * @param[in] buf line buffer * @param[in] len length of buffer * @param[in] offs offset where annotation starts * @returns 0 on success, something else otherwise */ static int processAnnotate(ln_ctx ctx, const char *buf, es_size_t lenBuf, es_size_t offs) { int r; es_str_t *tag = NULL; ln_annot *annot; ln_dbgprintf(ctx, "sample annotation to add: '%s'", buf+offs); CHKR(getFieldName(ctx, buf, lenBuf, &offs, &tag)); skipWhitespace(ctx, buf, lenBuf, &offs); if(buf[offs] != ':' || tag == NULL) { ln_dbgprintf(ctx, "invalid tag field in annotation, line is '%s'", buf); r=-1; goto done; } ++offs; /* we got an annotation! */ CHKN(annot = ln_newAnnot(tag)); while(offs < lenBuf) { CHKR(getAnnotationOp(ctx, annot, buf, lenBuf, &offs)); } r = ln_addAnnotToSet(ctx->pas, annot); done: return r; } struct ln_v1_samp * ln_v1_processSamp(ln_ctx ctx, const char *buf, es_size_t lenBuf) { struct ln_v1_samp *samp = NULL; es_str_t *typeStr = NULL; es_size_t offs; if(getLineType(buf, lenBuf, &offs, &typeStr) != 0) goto done; if(!es_strconstcmp(typeStr, "prefix")) { if(getPrefix(buf, lenBuf, offs, &ctx->rulePrefix) != 0) goto done; } else if(!es_strconstcmp(typeStr, "extendprefix")) { if(extendPrefix(ctx, buf, lenBuf, offs) != 0) goto done; } else if(!es_strconstcmp(typeStr, "rule")) { if(processRule(ctx, buf, lenBuf, offs) != 0) goto done; } else if(!es_strconstcmp(typeStr, "annotate")) { if(processAnnotate(ctx, buf, lenBuf, offs) != 0) goto done; } else { /* TODO error reporting */ char *str; str = es_str2cstr(typeStr, NULL); ln_dbgprintf(ctx, "invalid record type detected: '%s'", str); free(str); goto done; } done: if(typeStr != NULL) es_deleteStr(typeStr); return samp; } struct ln_v1_samp * ln_v1_sampRead(ln_ctx ctx, FILE *const __restrict__ repo, int *const __restrict__ isEof) { struct ln_v1_samp *samp = NULL; char buf[10*1024]; /**< max size of rule - TODO: make configurable */ size_t i = 0; int inParser = 0; int done = 0; while(!done) { int c = fgetc(repo); if(c == EOF) { *isEof = 1; if(i == 0) goto done; else done = 1; /* last line missing LF, still process it! */ } else if(c == '\n') { ++ctx->conf_ln_nbr; if(inParser) { if(ln_sampChkRunawayRule(ctx, repo, NULL)) { /* ignore previous rule */ inParser = 0; i = 0; } } if(!inParser && i != 0) done = 1; } else if(c == '#' && i == 0) { ln_sampSkipCommentLine(ctx, repo, NULL); i = 0; /* back to beginning */ } else { if(c == '%') inParser = (inParser) ? 0 : 1; buf[i++] = c; if(i >= sizeof(buf)) { ln_errprintf(ctx, 0, "line is too long"); goto done; } } } buf[i] = '\0'; ln_dbgprintf(ctx, "read rulebase line[~%d]: '%s'", ctx->conf_ln_nbr, buf); ln_v1_processSamp(ctx, buf, i); ln_dbgprintf(ctx, "---------------------------------------"); ln_displayPTree(ctx->ptree, 0); ln_dbgprintf(ctx, "======================================="); done: return samp; } liblognorm-2.0.8/src/v1_samp.h000066400000000000000000000063731511425433100162030ustar00rootroot00000000000000/** * @file samples.h * @brief Object to process log samples. * @author Rainer Gerhards * * This object handles log samples, and in actual log sample files. * It co-operates with the ptree object to build the actual parser tree. *//* * * liblognorm - a fast samples-based log normalization library * Copyright 2010 by Rainer Gerhards and Adiscon GmbH. * * This file is part of liblognorm. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution. */ #ifndef LIBLOGNORM_V1_SAMPLES_H_INCLUDED #define LIBLOGNORM_V1_SAMPLES_H_INCLUDED #include /* we need es_size_t */ #include /** * A single log sample. */ struct ln_v1_samp { es_str_t *msg; }; /** * Reads a sample stored in buffer buf and creates a new ln_v1_samp object * out of it. * * @note * It is the caller's responsibility to delete the newly * created ln_v1_samp object if it is no longer needed. * * @param[ctx] ctx current library context * @param[buf] cstr buffer containing the string contents of the sample * @param[lenBuf] length of the sample contained within buf * @return Newly create object or NULL if an error occurred. */ struct ln_v1_samp * ln_v1_processSamp(ln_ctx ctx, const char *buf, es_size_t lenBuf); /** * Read a sample from repository (sequentially). * * Reads a sample starting with the current file position and * creates a new ln_v1_samp object out of it. * * @note * It is the caller's responsibility to delete the newly * created ln_v1_samp object if it is no longer needed. * * @param[in] ctx current library context * @param[in] repo repository descriptor * @param[out] isEof must be set to 0 on entry and is switched to 1 if EOF occurred. * @return Newly create object or NULL if an error or EOF occurred. */ struct ln_v1_samp * ln_v1_sampRead(ln_ctx ctx, FILE *repo, int *isEof); /** * Free ln_v1_samp object. */ void ln_v1_sampFree(ln_ctx ctx, struct ln_v1_samp *samp); /** * Parse a given sample * * @param[in] ctx current library context * @param[in] rule string (with prefix and suffix '%' markers) * @param[in] offset in rule-string to start at (it should be pointed to * starting character: '%') * @param[in] temp string buffer(working space), * externalized for efficiency reasons * @param[out] return code (0 means success) * @return newly created node, which can be added to sample tree. */ ln_fieldList_t* ln_v1_parseFieldDescr(ln_ctx ctx, es_str_t *rule, es_size_t *bufOffs, es_str_t **str, int* ret); #endif /* #ifndef LIBLOGNORM_V1_SAMPLES_H_INCLUDED */ liblognorm-2.0.8/tests/000077500000000000000000000000001511425433100150265ustar00rootroot00000000000000liblognorm-2.0.8/tests/.gitignore000066400000000000000000000000731511425433100170160ustar00rootroot00000000000000*.o json_eq .deps core *.rulebase vgcore.* .libs user_test liblognorm-2.0.8/tests/Makefile.am000066400000000000000000000113121511425433100170600ustar00rootroot00000000000000check_PROGRAMS = json_eq # re-enable if we really need the c program check check_PROGRAMS = json_eq user_test json_eq_self_sources = json_eq.c json_eq_SOURCES = $(json_eq_self_sources) json_eq_CPPFLAGS = $(JSON_C_CFLAGS) $(WARN_CFLAGS) -I$(top_srcdir)/src json_eq_LDADD = $(JSON_C_LIBS) -lm -lestr json_eq_LDFLAGS = -no-install #user_test_SOURCES = user_test.c #user_test_CPPFLAGS = $(LIBLOGNORM_CFLAGS) $(JSON_C_CFLAGS) $(LIBESTR_CFLAGS) #user_test_LDADD = $(JSON_C_LIBS) $(LIBLOGNORM_LIBS) $(LIBESTR_LIBS) ../compat/compat.la #user_test_LDFLAGS = -no-install # The following tests are for the new pdag-based engine (v2+). # # There are some notes due: # # removed field_float_with_invalid_ruledef.sh because test is not valid. # more info: https://github.com/rsyslog/liblognorm/issues/98 # note that probably the other currently disable *_invalid_*.sh # tests are also affected. # # there seems to be a problem with some format in cisco-interface-spec # Probably this was just not seen in v1, because of some impreciseness # in the ptree normalizer. Pushing equivalent v2 test back until v2 # implementation is further developed. TESTS_SHELLSCRIPTS = \ usrdef_simple.sh \ usrdef_two.sh \ usrdef_twotypes.sh \ usrdef_actual1.sh \ usrdef_ipaddr.sh \ usrdef_ipaddr_dotdot.sh \ usrdef_ipaddr_dotdot2.sh \ usrdef_ipaddr_dotdot3.sh \ usrdef_nested_segfault.sh \ missing_line_ending.sh \ lognormalizer-invld-call.sh \ string_rb_simple.sh \ string_rb_simple_2_lines.sh \ names.sh \ literal.sh \ include.sh \ include_RULEBASES.sh \ seq_simple.sh \ runaway_rule.sh \ runaway_rule_comment.sh \ annotate.sh \ alternative_simple.sh \ alternative_three.sh \ alternative_nested.sh \ alternative_segfault.sh \ repeat_very_simple.sh \ repeat_simple.sh \ repeat_mismatch_in_while.sh \ repeat_while_alternative.sh \ repeat_alternative_nested.sh \ parser_prios.sh \ parser_whitespace.sh \ parser_whitespace_jsoncnf.sh \ parser_LF.sh \ parser_LF_jsoncnf.sh \ strict_prefix_actual_sample1.sh \ strict_prefix_matching_1.sh \ strict_prefix_matching_2.sh \ field_string.sh \ field_string_perm_chars.sh \ field_string_lazy_matching.sh \ field_string_doc_sample_lazy.sh \ field_number.sh \ field_number-fmt_number.sh \ field_number_maxval.sh \ field_hexnumber.sh \ field_hexnumber-fmt_number.sh \ field_hexnumber_jsoncnf.sh \ field_hexnumber_range.sh \ field_hexnumber_range_jsoncnf.sh \ rule_last_str_short.sh \ field_mac48.sh \ field_mac48_jsoncnf.sh \ field_name_value.sh \ field_name_value_jsoncnf.sh \ field_kernel_timestamp.sh \ field_kernel_timestamp_jsoncnf.sh \ field_whitespace.sh \ rule_last_str_long.sh \ field_whitespace_jsoncnf.sh \ field_rest.sh \ field_rest_jsoncnf.sh \ field_json.sh \ field_json_jsoncnf.sh \ field_cee-syslog.sh \ field_cee-syslog_jsoncnf.sh \ field_ipv6.sh \ field_ipv6_jsoncnf.sh \ field_v2-iptables.sh \ field_v2-iptables_jsoncnf.sh \ field_cef.sh \ field_cef_jsoncnf.sh \ field_checkpoint-lea.sh \ field_checkpoint-lea_jsoncnf.sh \ field_checkpoint-lea-terminator.sh \ field_duration.sh \ field_duration_jsoncnf.sh \ field_float.sh \ field_float-fmt_number.sh \ field_float_jsoncnf.sh \ field_rfc5424timestamp-fmt_timestamp-unix.sh \ field_rfc5424timestamp-fmt_timestamp-unix-ms.sh \ very_long_logline_jsoncnf.sh # now come tests for the legacy (v1) engine TESTS_SHELLSCRIPTS += \ missing_line_ending_v1.sh \ runaway_rule_v1.sh \ runaway_rule_comment_v1.sh \ field_hexnumber_v1.sh \ field_mac48_v1.sh \ field_name_value_v1.sh \ field_kernel_timestamp_v1.sh \ field_whitespace_v1.sh \ field_rest_v1.sh \ field_json_v1.sh \ field_cee-syslog_v1.sh \ field_ipv6_v1.sh \ field_v2-iptables_v1.sh \ field_cef_v1.sh \ field_checkpoint-lea_v1.sh \ field_duration_v1.sh \ field_float_v1.sh \ field_cee-syslog_v1.sh \ field_tokenized.sh \ field_tokenized_with_invalid_ruledef.sh \ field_recursive.sh \ field_tokenized_recursive.sh \ field_interpret.sh \ field_interpret_with_invalid_ruledef.sh \ field_descent.sh \ field_descent_with_invalid_ruledef.sh \ field_suffixed.sh \ field_suffixed_with_invalid_ruledef.sh \ field_cisco-interface-spec.sh \ field_cisco-interface-spec-at-EOL.sh \ field_float_with_invalid_ruledef.sh \ very_long_logline.sh #re-add to TESTS if needed: user_test TESTS = \ $(TESTS_SHELLSCRIPTS) REGEXP_TESTS = \ field_regex_default_group_parse_and_return.sh \ field_regex_invalid_args.sh \ field_regex_with_consume_group.sh \ field_regex_with_consume_group_and_return_group.sh \ field_regex_with_negation.sh \ field_tokenized_with_regex.sh \ field_regex_while_regex_support_is_disabled.sh EXTRA_DIST = exec.sh \ $(TESTS_SHELLSCRIPTS) \ $(REGEXP_TESTS) \ $(json_eq_self_sources) \ $(user_test_SOURCES) if ENABLE_REGEXP TESTS += $(REGEXP_TESTS) endif liblognorm-2.0.8/tests/alternative_nested.sh000077500000000000000000000011661511425433100212510ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple alternative syntax" add_rule 'version=2' add_rule 'rule=:a % {"type":"alternative", "parser": [ [ {"type":"number", "name":"num1"}, {"type":"literal", "text":":"}, {"type":"number", "name":"num"}, ], {"type":"hexnumber", "name":"hex"} ] }% b' execute 'a 47:11 b' assert_output_json_eq '{"num": "11", "num1": "47" }' execute 'a 0x4711 b' assert_output_json_eq '{ "hex": "0x4711" }' cleanup_tmp_files liblognorm-2.0.8/tests/alternative_segfault.sh000077500000000000000000000012771511425433100216040ustar00rootroot00000000000000#!/bin/bash # added 2016-10-17 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "a case that caused a segfault in practice" add_rule 'version=2' add_rule 'rule=:%host:ipv4% %{"type":"alternative","parser":[{"type":"literal","text":"-"},{"type":"number","name":"identd"}]}% %OK:word%' execute '1.2.3.4 - TEST_OK' assert_output_json_eq '{ "OK": "TEST_OK", "host": "1.2.3.4" }' execute '1.2.3.4 100 TEST_OK' assert_output_json_eq '{ "OK": "TEST_OK", "identd": "100", "host": "1.2.3.4" }' execute '1.2.3.4 ERR TEST_OK' assert_output_json_eq '{ "originalmsg": "1.2.3.4 ERR TEST_OK", "unparsed-data": "ERR TEST_OK" }' cleanup_tmp_files liblognorm-2.0.8/tests/alternative_simple.sh000077500000000000000000000007301511425433100212540ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple alternative syntax" add_rule 'version=2' add_rule 'rule=:a %{"type":"alternative", "parser":[{"name":"num", "type":"number"}, {"name":"hex", "type":"hexnumber"}]}% b' execute 'a 4711 b' assert_output_json_eq '{ "num": "4711" }' execute 'a 0x4711 b' assert_output_json_eq '{ "hex": "0x4711" }' cleanup_tmp_files liblognorm-2.0.8/tests/alternative_three.sh000077500000000000000000000010641511425433100210730ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple alternative syntax" add_rule 'version=2' add_rule 'rule=:a %{"type":"alternative", "parser":[{"name":"num", "type":"number"}, {"name":"hex", "type":"hexnumber"}, {"name":"wrd", "type":"word"}]}% b' execute 'a 4711 b' assert_output_json_eq '{ "num": "4711" }' execute 'a 0x4711 b' assert_output_json_eq '{ "hex": "0x4711" }' execute 'a 0xyz b' assert_output_json_eq '{ "wrd": "0xyz" }' cleanup_tmp_files liblognorm-2.0.8/tests/annotate.sh000077500000000000000000000015751511425433100172060ustar00rootroot00000000000000#!/bin/bash # added 2016-11-08 by Rainer Gerhards . $srcdir/exec.sh test_def $0 "annotate functionality" reset_rules add_rule 'version=2' add_rule 'rule=ABC,WIN:<%-:number%>1 %-:date-rfc5424% %-:word% %tag:word% - - -' add_rule 'rule=ABC:<%-:number%>1 %-:date-rfc5424% %-:word% %tag:word% + - -' add_rule 'rule=WIN:<%-:number%>1 %-:date-rfc5424% %-:word% %tag:word% . - -' add_rule 'annotate=WIN:+annot1="WIN" # inline-comment' add_rule 'annotate=ABC:+annot2="ABC"' execute '<37>1 2016-11-03T23:59:59+03:00 server.example.net TAG . - -' assert_output_json_eq '{ "tag": "TAG", "annot1": "WIN" }' execute '<37>1 2016-11-03T23:59:59+03:00 server.example.net TAG + - -' assert_output_json_eq '{ "tag": "TAG", "annot2": "ABC" }' execute '<6>1 2016-09-02T07:41:07+02:00 server.example.net TAG - - -' assert_output_json_eq '{ "tag": "TAG", "annot1": "WIN", "annot2": "ABC" }' cleanup_tmp_files liblognorm-2.0.8/tests/exec.sh000066400000000000000000000044271511425433100163150ustar00rootroot00000000000000# environment variables: # GREP - if set, can be used to use alternative grep version # Most important use case is to use GNU grep (ggrep) # on Solaris. If unset, use "grep". set -e if [ "x$debug" == "xon" ]; then #get core-dump on crash ulimit -c unlimited fi #cmd="../src/ln_test -v" # case to get debug info (add -vvv for more verbosity) cmd=../src/ln_test # regular case . ./options.sh no_solaris10() { if (uname -a | grep -q "SunOS.*5.10"); then printf 'platform: %s\n' "$(uname -a)" printf 'This looks like solaris 10, we disable known-failing tests to\n' printf 'permit OpenCSW to build packages. However, this are real failures\n' printf 'and so a fix should be done as soon as time permits.\n' exit 77 fi } test_def() { test_file=$(basename $1) test_name=$(echo $test_file | sed -e 's/\..*//g') echo =============================================================================== echo "[${test_file}]: test for ${2}" } execute() { if [ "x$debug" == "xon" ]; then echo "======rulebase=======" cat tmp.rulebase echo "=====================" set -x fi if [ "$1" == "file" ]; then $cmd $ln_opts -r tmp.rulebase -e json > test.out < $2 else echo "$1" | $cmd $ln_opts -r tmp.rulebase -e json > test.out fi echo "Out:" cat test.out if [ "x$debug" == "xon" ]; then set +x fi } execute_with_string() { # $1 must be rulebase string # $2 must be sample string if [ "x$debug" == "xon" ]; then echo "======rulebase=======" cat tmp.rulebase echo "=====================" set -x fi echo "$2" | $cmd $ln_opts -R "$1" -e json > test.out echo "Out:" cat test.out if [ "x$debug" == "xon" ]; then set +x fi } assert_output_contains() { ${GREP:-grep} -F "$1" < test.out } assert_output_json_eq() { ./json_eq "$1" "$(cat test.out)" } rulebase_file_name() { if [ "x$1" == "x" ]; then echo tmp.rulebase else echo $1.rulebase fi } reset_rules() { rb_file=$(rulebase_file_name $1) rm -f $rb_file } add_rule() { rb_file=$(rulebase_file_name $2) echo $1 >> $rb_file } add_rule_no_LF() { rb_file=$(rulebase_file_name $2) echo -n $1 >> $rb_file } cleanup_tmp_files() { rm -f -- test.out *.rulebase } reset_rules liblognorm-2.0.8/tests/field_cee-syslog.sh000077500000000000000000000015571511425433100206120ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "JSON field" add_rule 'version=2' add_rule 'rule=:%field:cee-syslog%' execute '@cee:{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '@cee:{"f1": "1", "f2": 2} ' # note the trailing space assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '@cee: {"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '@cee: {"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # # Things that MUST NOT work # execute '@cee: {"f1": "1", "f2": 2} data' assert_output_json_eq '{ "originalmsg": "@cee: {\"f1\": \"1\", \"f2\": 2} data", "unparsed-data": "@cee: {\"f1\": \"1\", \"f2\": 2} data" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_cee-syslog_jsoncnf.sh000077500000000000000000000016041511425433100223230ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "JSON field" add_rule 'version=2' add_rule 'rule=:%{"name":"field", "type":"cee-syslog"}%' execute '@cee:{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '@cee:{"f1": "1", "f2": 2} ' # note the trailing space assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '@cee: {"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '@cee: {"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # # Things that MUST NOT work # execute '@cee: {"f1": "1", "f2": 2} data' assert_output_json_eq '{ "originalmsg": "@cee: {\"f1\": \"1\", \"f2\": 2} data", "unparsed-data": "@cee: {\"f1\": \"1\", \"f2\": 2} data" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_cee-syslog_v1.sh000077500000000000000000000015321511425433100212110ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "JSON field" add_rule 'rule=:%field:cee-syslog%' execute '@cee:{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '@cee:{"f1": "1", "f2": 2} ' # note the trailing space assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '@cee: {"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '@cee: {"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # # Things that MUST NOT work # execute '@cee: {"f1": "1", "f2": 2} data' assert_output_json_eq '{ "originalmsg": "@cee: {\"f1\": \"1\", \"f2\": 2} data", "unparsed-data": "@cee: {\"f1\": \"1\", \"f2\": 2} data" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_cef.sh000077500000000000000000000231471511425433100172740ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "CEF parser" add_rule 'version=2' add_rule 'rule=:%f:cef%' # fabricated tests to test specific functionality execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a value cc=field 3' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a value", "cc": "field 3" } } }' execute 'CEF:0|Vendor|Product\|1\|\\|Version|Signature ID|some name|Severity| aa=field1 bb=this is a name\=value cc=field 3' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product|1|\\", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a name=value", "cc": "field 3" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \= value cc=field 3' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a = value", "cc": "field 3" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity|' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=value' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "name": "value" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=val\nue' # embedded LF assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "name": "val\nue" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| n,me=value' #invalid punctuation in extension assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| n,me=value", "unparsed-data": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| n,me=value" }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=v\alue' #invalid escape in extension assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=v\\alue", "unparsed-data": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=v\\alue" }' execute 'CEF:0|V\endor|Product|Version|Signature ID|some name|Severity| name=value' #invalid escape in header assert_output_json_eq '{ "originalmsg": "CEF:0|V\\endor|Product|Version|Signature ID|some name|Severity| name=value", "unparsed-data": "CEF:0|V\\endor|Product|Version|Signature ID|some name|Severity| name=value" }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| ' # single trailing space - valid assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| ' # multiple trailing spaces - invalid assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { } } }' execute 'CEF:0|Vendor' assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor", "unparsed-data": "CEF:0|Vendor" }' execute 'CEF:1|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \= value cc=field 3' assert_output_json_eq '{ "originalmsg": "CEF:1|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \\= value cc=field 3", "unparsed-data": "CEF:1|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \\= value cc=field 3" }' execute '' assert_output_json_eq '{ "originalmsg": "", "unparsed-data": "" }' # finally, a use case from practice execute 'CEF:0|ArcSight|ArcSight|10.0.0.15.0|rule:101|FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt|High| eventId=24934046519 type=2 mrt=8888882444085 sessionId=0 generatorID=34rSQWFOOOCAVlswcKFkbA\=\= categorySignificance=/Normal categoryBehavior=/Execute/Query categoryDeviceGroup=/Application categoryOutcome=/Success categoryObject=/Host/Application modelConfidence=0 severity=0 relevance=10 assetCriticality=0 priority=2 art=1427882454263 cat=/Detection/FOO/UNIX/Direct Root Connection Attempt deviceSeverity=Warning rt=1427881661000 shost=server.foo.bar src=10.0.0.1 sourceZoneID=MRL4p30sFOOO8panjcQnFbw\=\= sourceZoneURI=/All Zones/FOO Solutions/Server Subnet/UK/PAR-WDC-12-CELL5-PROD S2U 1 10.0.0.1-10.0.0.1 sourceGeoCountryCode=GB sourceGeoLocationInfo=London slong=-0.90843 slat=51.9039 dhost=server.foo.bar dst=10.0.0.1 destinationZoneID=McFOOO0sBABCUHR83pKJmQA\=\= destinationZoneURI=/All Zones/FOO Solutions/Prod/AMERICAS/FOO 10.0.0.1-10.0.0.1 duser=johndoe destinationGeoCountryCode=US destinationGeoLocationInfo=Jersey City dlong=-90.0435 dlat=30.732 fname=FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt filePath=/All Rules/Real-time Rules/ACBP-ACCESS CONTROL and AUTHORIZATION/FOO/Unix Server/FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt fileType=Rule ruleThreadId=NQVtdFOOABDrKsmLWpyq8g\=\= cs2= flexString2=DC0001-988 locality=1 cs2Label=Configuration Resource ahost=foo.bar agt=10.0.0.1 av=10.0.0.12 atz=Europe/Berlin aid=34rSQWFOOOBCAVlswcKFkbA\=\= at=superagent_ng dvchost=server.foo.bar dvc=10.0.0.1 deviceZoneID=Mbb8pFOOODol1dBKdURJA\=\= deviceZoneURI=/All Zones/FOO Solutions/Prod/GERMANY/FOO US2 Class6 A 508 10.0.0.1-10.0.0.1 dtz=Europe/Berlin deviceFacility=Rules Engine eventAnnotationStageUpdateTime=1427882444192 eventAnnotationModificationTime=1427882444192 eventAnnotationAuditTrail=1,1427453188050,root,Queued,,,,\n eventAnnotationVersion=1 eventAnnotationFlags=0 eventAnnotationEndTime=1427881661000 eventAnnotationManagerReceiptTime=1427882444085 _cefVer=0.1 ad.arcSightEventPath=3VcygrkkBABCAYFOOLlU13A\=\= baseEventIds=24934003731"' assert_output_json_eq '{ "f": { "DeviceVendor": "ArcSight", "DeviceProduct": "ArcSight", "DeviceVersion": "10.0.0.15.0", "SignatureID": "rule:101", "Name": "FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt", "Severity": "High", "Extensions": { "eventId": "24934046519", "type": "2", "mrt": "8888882444085", "sessionId": "0", "generatorID": "34rSQWFOOOCAVlswcKFkbA==", "categorySignificance": "\/Normal", "categoryBehavior": "\/Execute\/Query", "categoryDeviceGroup": "\/Application", "categoryOutcome": "\/Success", "categoryObject": "\/Host\/Application", "modelConfidence": "0", "severity": "0", "relevance": "10", "assetCriticality": "0", "priority": "2", "art": "1427882454263", "cat": "\/Detection\/FOO\/UNIX\/Direct Root Connection Attempt", "deviceSeverity": "Warning", "rt": "1427881661000", "shost": "server.foo.bar", "src": "10.0.0.1", "sourceZoneID": "MRL4p30sFOOO8panjcQnFbw==", "sourceZoneURI": "\/All Zones\/FOO Solutions\/Server Subnet\/UK\/PAR-WDC-12-CELL5-PROD S2U 1 10.0.0.1-10.0.0.1", "sourceGeoCountryCode": "GB", "sourceGeoLocationInfo": "London", "slong": "-0.90843", "slat": "51.9039", "dhost": "server.foo.bar", "dst": "10.0.0.1", "destinationZoneID": "McFOOO0sBABCUHR83pKJmQA==", "destinationZoneURI": "\/All Zones\/FOO Solutions\/Prod\/AMERICAS\/FOO 10.0.0.1-10.0.0.1", "duser": "johndoe", "destinationGeoCountryCode": "US", "destinationGeoLocationInfo": "Jersey City", "dlong": "-90.0435", "dlat": "30.732", "fname": "FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt", "filePath": "\/All Rules\/Real-time Rules\/ACBP-ACCESS CONTROL and AUTHORIZATION\/FOO\/Unix Server\/FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt", "fileType": "Rule", "ruleThreadId": "NQVtdFOOABDrKsmLWpyq8g==", "cs2": "", "flexString2": "DC0001-988", "locality": "1", "cs2Label": "Configuration Resource", "ahost": "foo.bar", "agt": "10.0.0.1", "av": "10.0.0.12", "atz": "Europe\/Berlin", "aid": "34rSQWFOOOBCAVlswcKFkbA==", "at": "superagent_ng", "dvchost": "server.foo.bar", "dvc": "10.0.0.1", "deviceZoneID": "Mbb8pFOOODol1dBKdURJA==", "deviceZoneURI": "\/All Zones\/FOO Solutions\/Prod\/GERMANY\/FOO US2 Class6 A 508 10.0.0.1-10.0.0.1", "dtz": "Europe\/Berlin", "deviceFacility": "Rules Engine", "eventAnnotationStageUpdateTime": "1427882444192", "eventAnnotationModificationTime": "1427882444192", "eventAnnotationAuditTrail": "1,1427453188050,root,Queued,,,,\n", "eventAnnotationVersion": "1", "eventAnnotationFlags": "0", "eventAnnotationEndTime": "1427881661000", "eventAnnotationManagerReceiptTime": "1427882444085", "_cefVer": "0.1", "ad.arcSightEventPath": "3VcygrkkBABCAYFOOLlU13A==", "baseEventIds": "24934003731\"" } } }' cleanup_tmp_files liblognorm-2.0.8/tests/field_cef_jsoncnf.sh000077500000000000000000000231741511425433100210140ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "CEF parser" add_rule 'version=2' add_rule 'rule=:%{"name":"f", "type":"cef"}%' # fabricated tests to test specific functionality execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a value cc=field 3' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a value", "cc": "field 3" } } }' execute 'CEF:0|Vendor|Product\|1\|\\|Version|Signature ID|some name|Severity| aa=field1 bb=this is a name\=value cc=field 3' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product|1|\\", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a name=value", "cc": "field 3" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \= value cc=field 3' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a = value", "cc": "field 3" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity|' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=value' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "name": "value" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=val\nue' # embedded LF assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "name": "val\nue" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| n,me=value' #invalid punctuation in extension assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| n,me=value", "unparsed-data": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| n,me=value" }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=v\alue' #invalid escape in extension assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=v\\alue", "unparsed-data": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=v\\alue" }' execute 'CEF:0|V\endor|Product|Version|Signature ID|some name|Severity| name=value' #invalid escape in header assert_output_json_eq '{ "originalmsg": "CEF:0|V\\endor|Product|Version|Signature ID|some name|Severity| name=value", "unparsed-data": "CEF:0|V\\endor|Product|Version|Signature ID|some name|Severity| name=value" }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| ' # single trailing space - valid assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| ' # multiple trailing spaces - invalid assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { } } }' execute 'CEF:0|Vendor' assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor", "unparsed-data": "CEF:0|Vendor" }' execute 'CEF:1|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \= value cc=field 3' assert_output_json_eq '{ "originalmsg": "CEF:1|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \\= value cc=field 3", "unparsed-data": "CEF:1|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \\= value cc=field 3" }' execute '' assert_output_json_eq '{ "originalmsg": "", "unparsed-data": "" }' # finally, a use case from practice execute 'CEF:0|ArcSight|ArcSight|10.0.0.15.0|rule:101|FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt|High| eventId=24934046519 type=2 mrt=8888882444085 sessionId=0 generatorID=34rSQWFOOOCAVlswcKFkbA\=\= categorySignificance=/Normal categoryBehavior=/Execute/Query categoryDeviceGroup=/Application categoryOutcome=/Success categoryObject=/Host/Application modelConfidence=0 severity=0 relevance=10 assetCriticality=0 priority=2 art=1427882454263 cat=/Detection/FOO/UNIX/Direct Root Connection Attempt deviceSeverity=Warning rt=1427881661000 shost=server.foo.bar src=10.0.0.1 sourceZoneID=MRL4p30sFOOO8panjcQnFbw\=\= sourceZoneURI=/All Zones/FOO Solutions/Server Subnet/UK/PAR-WDC-12-CELL5-PROD S2U 1 10.0.0.1-10.0.0.1 sourceGeoCountryCode=GB sourceGeoLocationInfo=London slong=-0.90843 slat=51.9039 dhost=server.foo.bar dst=10.0.0.1 destinationZoneID=McFOOO0sBABCUHR83pKJmQA\=\= destinationZoneURI=/All Zones/FOO Solutions/Prod/AMERICAS/FOO 10.0.0.1-10.0.0.1 duser=johndoe destinationGeoCountryCode=US destinationGeoLocationInfo=Jersey City dlong=-90.0435 dlat=30.732 fname=FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt filePath=/All Rules/Real-time Rules/ACBP-ACCESS CONTROL and AUTHORIZATION/FOO/Unix Server/FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt fileType=Rule ruleThreadId=NQVtdFOOABDrKsmLWpyq8g\=\= cs2= flexString2=DC0001-988 locality=1 cs2Label=Configuration Resource ahost=foo.bar agt=10.0.0.1 av=10.0.0.12 atz=Europe/Berlin aid=34rSQWFOOOBCAVlswcKFkbA\=\= at=superagent_ng dvchost=server.foo.bar dvc=10.0.0.1 deviceZoneID=Mbb8pFOOODol1dBKdURJA\=\= deviceZoneURI=/All Zones/FOO Solutions/Prod/GERMANY/FOO US2 Class6 A 508 10.0.0.1-10.0.0.1 dtz=Europe/Berlin deviceFacility=Rules Engine eventAnnotationStageUpdateTime=1427882444192 eventAnnotationModificationTime=1427882444192 eventAnnotationAuditTrail=1,1427453188050,root,Queued,,,,\n eventAnnotationVersion=1 eventAnnotationFlags=0 eventAnnotationEndTime=1427881661000 eventAnnotationManagerReceiptTime=1427882444085 _cefVer=0.1 ad.arcSightEventPath=3VcygrkkBABCAYFOOLlU13A\=\= baseEventIds=24934003731"' assert_output_json_eq '{ "f": { "DeviceVendor": "ArcSight", "DeviceProduct": "ArcSight", "DeviceVersion": "10.0.0.15.0", "SignatureID": "rule:101", "Name": "FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt", "Severity": "High", "Extensions": { "eventId": "24934046519", "type": "2", "mrt": "8888882444085", "sessionId": "0", "generatorID": "34rSQWFOOOCAVlswcKFkbA==", "categorySignificance": "\/Normal", "categoryBehavior": "\/Execute\/Query", "categoryDeviceGroup": "\/Application", "categoryOutcome": "\/Success", "categoryObject": "\/Host\/Application", "modelConfidence": "0", "severity": "0", "relevance": "10", "assetCriticality": "0", "priority": "2", "art": "1427882454263", "cat": "\/Detection\/FOO\/UNIX\/Direct Root Connection Attempt", "deviceSeverity": "Warning", "rt": "1427881661000", "shost": "server.foo.bar", "src": "10.0.0.1", "sourceZoneID": "MRL4p30sFOOO8panjcQnFbw==", "sourceZoneURI": "\/All Zones\/FOO Solutions\/Server Subnet\/UK\/PAR-WDC-12-CELL5-PROD S2U 1 10.0.0.1-10.0.0.1", "sourceGeoCountryCode": "GB", "sourceGeoLocationInfo": "London", "slong": "-0.90843", "slat": "51.9039", "dhost": "server.foo.bar", "dst": "10.0.0.1", "destinationZoneID": "McFOOO0sBABCUHR83pKJmQA==", "destinationZoneURI": "\/All Zones\/FOO Solutions\/Prod\/AMERICAS\/FOO 10.0.0.1-10.0.0.1", "duser": "johndoe", "destinationGeoCountryCode": "US", "destinationGeoLocationInfo": "Jersey City", "dlong": "-90.0435", "dlat": "30.732", "fname": "FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt", "filePath": "\/All Rules\/Real-time Rules\/ACBP-ACCESS CONTROL and AUTHORIZATION\/FOO\/Unix Server\/FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt", "fileType": "Rule", "ruleThreadId": "NQVtdFOOABDrKsmLWpyq8g==", "cs2": "", "flexString2": "DC0001-988", "locality": "1", "cs2Label": "Configuration Resource", "ahost": "foo.bar", "agt": "10.0.0.1", "av": "10.0.0.12", "atz": "Europe\/Berlin", "aid": "34rSQWFOOOBCAVlswcKFkbA==", "at": "superagent_ng", "dvchost": "server.foo.bar", "dvc": "10.0.0.1", "deviceZoneID": "Mbb8pFOOODol1dBKdURJA==", "deviceZoneURI": "\/All Zones\/FOO Solutions\/Prod\/GERMANY\/FOO US2 Class6 A 508 10.0.0.1-10.0.0.1", "dtz": "Europe\/Berlin", "deviceFacility": "Rules Engine", "eventAnnotationStageUpdateTime": "1427882444192", "eventAnnotationModificationTime": "1427882444192", "eventAnnotationAuditTrail": "1,1427453188050,root,Queued,,,,\n", "eventAnnotationVersion": "1", "eventAnnotationFlags": "0", "eventAnnotationEndTime": "1427881661000", "eventAnnotationManagerReceiptTime": "1427882444085", "_cefVer": "0.1", "ad.arcSightEventPath": "3VcygrkkBABCAYFOOLlU13A==", "baseEventIds": "24934003731\"" } } }' cleanup_tmp_files liblognorm-2.0.8/tests/field_cef_v1.sh000077500000000000000000000231001511425433100176670ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "CEF parser" add_rule 'rule=:%f:cef%' # fabricated tests to test specific functionality execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a value cc=field 3' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a value", "cc": "field 3" } } }' execute 'CEF:0|Vendor|Product\|1\|\\|Version|Signature ID|some name|Severity| aa=field1 bb=this is a name\=value cc=field 3' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product|1|\\", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a name=value", "cc": "field 3" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \= value cc=field 3' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "aa": "field1", "bb": "this is a = value", "cc": "field 3" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity|' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=value' assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "name": "value" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=val\nue' # embedded LF assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { "name": "val\nue" } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| n,me=value' #invalid punctuation in extension assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| n,me=value", "unparsed-data": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| n,me=value" }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=v\alue' #invalid escape in extension assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=v\\alue", "unparsed-data": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| name=v\\alue" }' execute 'CEF:0|V\endor|Product|Version|Signature ID|some name|Severity| name=value' #invalid escape in header assert_output_json_eq '{ "originalmsg": "CEF:0|V\\endor|Product|Version|Signature ID|some name|Severity| name=value", "unparsed-data": "CEF:0|V\\endor|Product|Version|Signature ID|some name|Severity| name=value" }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| ' # single trailing space - valid assert_output_json_eq '{ "f": { "DeviceVendor": "Vendor", "DeviceProduct": "Product", "DeviceVersion": "Version", "SignatureID": "Signature ID", "Name": "some name", "Severity": "Severity", "Extensions": { } } }' execute 'CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| ' # multiple trailing spaces - invalid assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| ", "unparsed-data": "CEF:0|Vendor|Product|Version|Signature ID|some name|Severity| " }' execute 'CEF:0|Vendor' assert_output_json_eq '{ "originalmsg": "CEF:0|Vendor", "unparsed-data": "CEF:0|Vendor" }' execute 'CEF:1|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \= value cc=field 3' assert_output_json_eq '{ "originalmsg": "CEF:1|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \\= value cc=field 3", "unparsed-data": "CEF:1|Vendor|Product|Version|Signature ID|some name|Severity| aa=field1 bb=this is a \\= value cc=field 3" }' execute '' assert_output_json_eq '{ "originalmsg": "", "unparsed-data": "" }' # finally, a use case from practice execute 'CEF:0|ArcSight|ArcSight|10.0.0.15.0|rule:101|FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt|High| eventId=24934046519 type=2 mrt=8888882444085 sessionId=0 generatorID=34rSQWFOOOCAVlswcKFkbA\=\= categorySignificance=/Normal categoryBehavior=/Execute/Query categoryDeviceGroup=/Application categoryOutcome=/Success categoryObject=/Host/Application modelConfidence=0 severity=0 relevance=10 assetCriticality=0 priority=2 art=1427882454263 cat=/Detection/FOO/UNIX/Direct Root Connection Attempt deviceSeverity=Warning rt=1427881661000 shost=server.foo.bar src=10.0.0.1 sourceZoneID=MRL4p30sFOOO8panjcQnFbw\=\= sourceZoneURI=/All Zones/FOO Solutions/Server Subnet/UK/PAR-WDC-12-CELL5-PROD S2U 1 10.0.0.1-10.0.0.1 sourceGeoCountryCode=GB sourceGeoLocationInfo=London slong=-0.90843 slat=51.9039 dhost=server.foo.bar dst=10.0.0.1 destinationZoneID=McFOOO0sBABCUHR83pKJmQA\=\= destinationZoneURI=/All Zones/FOO Solutions/Prod/AMERICAS/FOO 10.0.0.1-10.0.0.1 duser=johndoe destinationGeoCountryCode=US destinationGeoLocationInfo=Jersey City dlong=-90.0435 dlat=30.732 fname=FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt filePath=/All Rules/Real-time Rules/ACBP-ACCESS CONTROL and AUTHORIZATION/FOO/Unix Server/FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt fileType=Rule ruleThreadId=NQVtdFOOABDrKsmLWpyq8g\=\= cs2= flexString2=DC0001-988 locality=1 cs2Label=Configuration Resource ahost=foo.bar agt=10.0.0.1 av=10.0.0.12 atz=Europe/Berlin aid=34rSQWFOOOBCAVlswcKFkbA\=\= at=superagent_ng dvchost=server.foo.bar dvc=10.0.0.1 deviceZoneID=Mbb8pFOOODol1dBKdURJA\=\= deviceZoneURI=/All Zones/FOO Solutions/Prod/GERMANY/FOO US2 Class6 A 508 10.0.0.1-10.0.0.1 dtz=Europe/Berlin deviceFacility=Rules Engine eventAnnotationStageUpdateTime=1427882444192 eventAnnotationModificationTime=1427882444192 eventAnnotationAuditTrail=1,1427453188050,root,Queued,,,,\n eventAnnotationVersion=1 eventAnnotationFlags=0 eventAnnotationEndTime=1427881661000 eventAnnotationManagerReceiptTime=1427882444085 _cefVer=0.1 ad.arcSightEventPath=3VcygrkkBABCAYFOOLlU13A\=\= baseEventIds=24934003731"' assert_output_json_eq '{ "f": { "DeviceVendor": "ArcSight", "DeviceProduct": "ArcSight", "DeviceVersion": "10.0.0.15.0", "SignatureID": "rule:101", "Name": "FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt", "Severity": "High", "Extensions": { "eventId": "24934046519", "type": "2", "mrt": "8888882444085", "sessionId": "0", "generatorID": "34rSQWFOOOCAVlswcKFkbA==", "categorySignificance": "\/Normal", "categoryBehavior": "\/Execute\/Query", "categoryDeviceGroup": "\/Application", "categoryOutcome": "\/Success", "categoryObject": "\/Host\/Application", "modelConfidence": "0", "severity": "0", "relevance": "10", "assetCriticality": "0", "priority": "2", "art": "1427882454263", "cat": "\/Detection\/FOO\/UNIX\/Direct Root Connection Attempt", "deviceSeverity": "Warning", "rt": "1427881661000", "shost": "server.foo.bar", "src": "10.0.0.1", "sourceZoneID": "MRL4p30sFOOO8panjcQnFbw==", "sourceZoneURI": "\/All Zones\/FOO Solutions\/Server Subnet\/UK\/PAR-WDC-12-CELL5-PROD S2U 1 10.0.0.1-10.0.0.1", "sourceGeoCountryCode": "GB", "sourceGeoLocationInfo": "London", "slong": "-0.90843", "slat": "51.9039", "dhost": "server.foo.bar", "dst": "10.0.0.1", "destinationZoneID": "McFOOO0sBABCUHR83pKJmQA==", "destinationZoneURI": "\/All Zones\/FOO Solutions\/Prod\/AMERICAS\/FOO 10.0.0.1-10.0.0.1", "duser": "johndoe", "destinationGeoCountryCode": "US", "destinationGeoLocationInfo": "Jersey City", "dlong": "-90.0435", "dlat": "30.732", "fname": "FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt", "filePath": "\/All Rules\/Real-time Rules\/ACBP-ACCESS CONTROL and AUTHORIZATION\/FOO\/Unix Server\/FOO-UNIX-Bypassing Golden Host-Direct Root Connection Attempt", "fileType": "Rule", "ruleThreadId": "NQVtdFOOABDrKsmLWpyq8g==", "cs2": "", "flexString2": "DC0001-988", "locality": "1", "cs2Label": "Configuration Resource", "ahost": "foo.bar", "agt": "10.0.0.1", "av": "10.0.0.12", "atz": "Europe\/Berlin", "aid": "34rSQWFOOOBCAVlswcKFkbA==", "at": "superagent_ng", "dvchost": "server.foo.bar", "dvc": "10.0.0.1", "deviceZoneID": "Mbb8pFOOODol1dBKdURJA==", "deviceZoneURI": "\/All Zones\/FOO Solutions\/Prod\/GERMANY\/FOO US2 Class6 A 508 10.0.0.1-10.0.0.1", "dtz": "Europe\/Berlin", "deviceFacility": "Rules Engine", "eventAnnotationStageUpdateTime": "1427882444192", "eventAnnotationModificationTime": "1427882444192", "eventAnnotationAuditTrail": "1,1427453188050,root,Queued,,,,\n", "eventAnnotationVersion": "1", "eventAnnotationFlags": "0", "eventAnnotationEndTime": "1427881661000", "eventAnnotationManagerReceiptTime": "1427882444085", "_cefVer": "0.1", "ad.arcSightEventPath": "3VcygrkkBABCAYFOOLlU13A==", "baseEventIds": "24934003731\"" } } }' cleanup_tmp_files liblognorm-2.0.8/tests/field_checkpoint-lea-terminator.sh000077500000000000000000000006571511425433100236100ustar00rootroot00000000000000#!/bin/bash # added 2018-10-31 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "Checkpoint LEA parser" add_rule 'version=2' add_rule 'rule=:[ %{"name":"f", "type":"checkpoint-lea", "terminator": "]"}%]' execute '[ tcp_flags: RST-ACK; src: 192.168.0.1; ]' assert_output_json_eq '{ "f": { "tcp_flags": "RST-ACK", "src": "192.168.0.1" } }' cleanup_tmp_files liblognorm-2.0.8/tests/field_checkpoint-lea.sh000077500000000000000000000006001511425433100214120ustar00rootroot00000000000000#!/bin/bash # added 2015-06-18 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "Checkpoint LEA parser" add_rule 'version=2' add_rule 'rule=:%f:checkpoint-lea%' execute 'tcp_flags: RST-ACK; src: 192.168.0.1;' assert_output_json_eq '{ "f": { "tcp_flags": "RST-ACK", "src": "192.168.0.1" } }' cleanup_tmp_files liblognorm-2.0.8/tests/field_checkpoint-lea_jsoncnf.sh000077500000000000000000000006251511425433100231410ustar00rootroot00000000000000#!/bin/bash # added 2015-06-18 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "Checkpoint LEA parser" add_rule 'version=2' add_rule 'rule=:%{"name":"f", "type":"checkpoint-lea"}%' execute 'tcp_flags: RST-ACK; src: 192.168.0.1;' assert_output_json_eq '{ "f": { "tcp_flags": "RST-ACK", "src": "192.168.0.1" } }' cleanup_tmp_files liblognorm-2.0.8/tests/field_checkpoint-lea_v1.sh000077500000000000000000000005531511425433100220270ustar00rootroot00000000000000#!/bin/bash # added 2015-06-18 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "Checkpoint LEA parser" add_rule 'rule=:%f:checkpoint-lea%' execute 'tcp_flags: RST-ACK; src: 192.168.0.1;' assert_output_json_eq '{ "f": { "tcp_flags": "RST-ACK", "src": "192.168.0.1" } }' cleanup_tmp_files liblognorm-2.0.8/tests/field_cisco-interface-spec-at-EOL.sh000077500000000000000000000011241511425433100235330ustar00rootroot00000000000000#!/bin/bash # added 2015-04-13 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "cisco-interface-spec-at-EOL syntax" add_rule 'version=2' add_rule 'rule=:begin %field:cisco-interface-spec%%r:rest%' execute 'begin outside:192.0.2.1/50349 end' assert_output_json_eq '{ "r": " end", "field": { "interface": "outside", "ip": "192.0.2.1", "port": "50349" } }' execute 'begin outside:192.0.2.1/50349' assert_output_json_eq '{ "r": "", "field": { "interface": "outside", "ip": "192.0.2.1", "port": "50349" } }' cleanup_tmp_files liblognorm-2.0.8/tests/field_cisco-interface-spec.sh000077500000000000000000000045541511425433100225260ustar00rootroot00000000000000#!/bin/bash # added 2015-04-13 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "cisco-interface-spec syntax" add_rule 'rule=:begin %field:cisco-interface-spec% end' execute 'begin outside:176.97.252.102/50349 end' assert_output_json_eq '{"field": { "interface": "outside", "ip": "176.97.252.102", "port": "50349" } }' execute 'begin outside:176.97.252.102/50349(DOMAIN\rainer) end' # we need to add the backslash escape for the testbench plumbing assert_output_json_eq '{"field": { "interface": "outside", "ip": "176.97.252.102", "port": "50349", "user": "DOMAIN\\rainer" } }' execute 'begin outside:176.97.252.102/50349(test/rainer) end' # we need to add the backslash escape for the testbench plumbing assert_output_json_eq '{"field": { "interface": "outside", "ip": "176.97.252.102", "port": "50349", "user": "test/rainer" } }' execute 'begin outside:176.97.252.102/50349(rainer) end' # we need to add the backslash escape for the testbench plumbing assert_output_json_eq '{"field": { "interface": "outside", "ip": "176.97.252.102", "port": "50349", "user": "rainer" } }' execute 'begin outside:192.168.1.13/50179 (192.168.1.13/50179)(LOCAL\some.user) end' assert_output_json_eq ' { "field": { "interface": "outside", "ip": "192.168.1.13", "port": "50179", "ip2": "192.168.1.13", "port2": "50179", "user": "LOCAL\\some.user" } }' execute 'begin outside:192.168.1.13/50179 (192.168.1.13/50179) (LOCAL\some.user) end' assert_output_json_eq ' { "field": { "interface": "outside", "ip": "192.168.1.13", "port": "50179", "ip2": "192.168.1.13", "port2": "50179", "user": "LOCAL\\some.user" } }' execute 'begin 192.168.1.13/50179 (192.168.1.13/50179) (LOCAL\without.if) end' assert_output_json_eq ' { "field": { "ip": "192.168.1.13", "port": "50179", "ip2": "192.168.1.13", "port2": "50179", "user": "LOCAL\\without.if" } }' # # Test for things that MUST NOT match! # # the SP before the second IP is missing: execute 'begin outside:192.168.1.13/50179(192.168.1.13/50179)(LOCAL\some.user) end' # note: the expected result looks a bit strange. This is the case because we # cannot (yet?) detect that "(192.168.1.13/50179)" is not a valid user name. assert_output_json_eq '{ "originalmsg": "begin outside:192.168.1.13\/50179(192.168.1.13\/50179)(LOCAL\\some.user) end", "unparsed-data": "(LOCAL\\some.user) end" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_descent.sh000077500000000000000000000102741511425433100201610ustar00rootroot00000000000000#!/bin/bash # added 2014-12-11 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "descent based parsing field" #descent with default tail field add_rule 'rule=:blocked on %device:word% %net:descent:./child.rulebase%at %tm:date-rfc5424%' reset_rules 'child' add_rule 'rule=:%ip_addr:ipv4% %tail:rest%' 'child' add_rule 'rule=:%subnet_addr:ipv4%/%mask:number% %tail:rest%' 'child' execute 'blocked on gw-1 10.20.30.40 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"ip_addr": "10.20.30.40"}, "tm": "2014-12-08T08:53:33.05+05:30"}' execute 'blocked on gw-1 10.20.30.40/16 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"subnet_addr": "10.20.30.40", "mask": "16"}, "tm": "2014-12-08T08:53:33.05+05:30"}' #descent with tail field being explicitly named 'tail' reset_rules add_rule 'rule=:blocked on %device:word% %net:descent:./field.rulebase:tail%at %tm:date-rfc5424%' reset_rules 'field' add_rule 'rule=:%ip_addr:ipv4% %tail:rest%' 'field' add_rule 'rule=:%subnet_addr:ipv4%/%mask:number% %tail:rest%' 'field' execute 'blocked on gw-1 10.20.30.40 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"ip_addr": "10.20.30.40"}, "tm": "2014-12-08T08:53:33.05+05:30"}' execute 'blocked on gw-1 10.20.30.40/16 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"subnet_addr": "10.20.30.40", "mask": "16"}, "tm": "2014-12-08T08:53:33.05+05:30"}' #descent with tail field having arbitrary name reset_rules add_rule 'rule=:blocked on %device:word% %net:descent:./subset.rulebase:remaining%at %tm:date-rfc5424%' reset_rules 'subset' add_rule 'rule=:%ip_addr:ipv4% %remaining:rest%' 'subset' add_rule 'rule=:%subnet_addr:ipv4%/%mask:number% %remaining:rest%' 'subset' execute 'blocked on gw-1 10.20.30.40 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"ip_addr": "10.20.30.40"}, "tm": "2014-12-08T08:53:33.05+05:30"}' execute 'blocked on gw-1 10.20.30.40/16 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"subnet_addr": "10.20.30.40", "mask": "16"}, "tm": "2014-12-08T08:53:33.05+05:30"}' #head call handling with with separate rulebase and tail field with with arbitrary name (this is what recursive field can't do) reset_rules add_rule 'rule=:%net:descent:./alt.rulebase:remains%blocked on %device:word%' reset_rules 'alt' add_rule 'rule=:%ip_addr:ipv4% %remains:rest%' 'alt' add_rule 'rule=:%subnet_addr:ipv4%/%mask:number% %remains:rest%' 'alt' execute '10.20.30.40 blocked on gw-1' assert_output_json_eq '{"device": "gw-1", "net": {"ip_addr": "10.20.30.40"}}' execute '10.20.30.40/16 blocked on gw-1' assert_output_json_eq '{"device": "gw-1", "net": {"subnet_addr": "10.20.30.40", "mask": "16"}}' #descent-field which calls another descent-field reset_rules add_rule 'rule=:%op:descent:./op.rulebase:rest% on %device:word%' reset_rules 'op' add_rule 'rule=:%net:descent:./alt.rulebase:remains%%action:word%%rest:rest%' 'op' reset_rules 'alt' add_rule 'rule=:%ip_addr:ipv4% %remains:rest%' 'alt' add_rule 'rule=:%subnet_addr:ipv4%/%mask:number% %remains:rest%' 'alt' execute '10.20.30.40 blocked on gw-1' assert_output_json_eq '{"op": {"action": "blocked", "net": {"ip_addr": "10.20.30.40"}}, "device": "gw-1"}' execute '10.20.30.40/16 unblocked on gw-2' assert_output_json_eq '{"op": {"action": "unblocked", "net": {"subnet_addr": "10.20.30.40", "mask": "16"}}, "device": "gw-2"}' #descent with file name having lognorm special char add_rule 'rule=:blocked on %device:word% %net:descent:./part\x3anet.rulebase%at %tm:date-rfc5424%' reset_rules 'part:net' add_rule 'rule=:%ip_addr:ipv4% %tail:rest%' 'part:net' add_rule 'rule=:%subnet_addr:ipv4%/%mask:number% %tail:rest%' 'part:net' execute 'blocked on gw-1 10.20.30.40 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"ip_addr": "10.20.30.40"}, "tm": "2014-12-08T08:53:33.05+05:30"}' execute 'blocked on gw-1 10.20.30.40/16 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"subnet_addr": "10.20.30.40", "mask": "16"}, "tm": "2014-12-08T08:53:33.05+05:30"}' cleanup_tmp_files liblognorm-2.0.8/tests/field_descent_with_invalid_ruledef.sh000077500000000000000000000051071511425433100244270ustar00rootroot00000000000000#!/bin/bash # added 2014-12-15 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "descent based parsing field, with invalid ruledef" #invalid parent field name add_rule 'rule=:%net:desce%' execute '10.20.30.40 foo' assert_output_json_eq '{ "originalmsg": "10.20.30.40 foo", "unparsed-data": "10.20.30.40 foo" }' #no args add_rule 'rule=:%net:descent%' execute '10.20.30.40 foo' assert_output_json_eq '{ "originalmsg": "10.20.30.40 foo", "unparsed-data": "10.20.30.40 foo" }' #incorrect rulebase file path rm -f $srcdir/quux.rulebase add_rule 'rule=:%net:descent:./quux.rulebase%' execute '10.20.30.40 foo' assert_output_json_eq '{ "originalmsg": "10.20.30.40 foo", "unparsed-data": "10.20.30.40 foo" }' #invalid content in rulebase file reset_rules add_rule 'rule=:%net:descent:./child.rulebase%' reset_rules 'child' add_rule 'rule=:%ip_addr:ipv4 %tail:rest%' 'child' execute '10.20.30.40 foo' assert_output_json_eq '{ "originalmsg": "10.20.30.40 foo", "unparsed-data": "10.20.30.40 foo" }' #empty child rulebase file reset_rules add_rule 'rule=:%net:descent:./child.rulebase%' reset_rules 'child' execute '10.20.30.40 foo' assert_output_json_eq '{ "originalmsg": "10.20.30.40 foo", "unparsed-data": "10.20.30.40 foo" }' #no rulebase given reset_rules add_rule 'rule=:%net:descent:' reset_rules 'child' execute '10.20.30.40 foo' assert_output_json_eq '{ "originalmsg": "10.20.30.40 foo", "unparsed-data": "10.20.30.40 foo" }' #no rulebase and no tail-field given reset_rules add_rule 'rule=:%net:descent::' reset_rules 'child' execute '10.20.30.40 foo' assert_output_json_eq '{ "originalmsg": "10.20.30.40 foo", "unparsed-data": "10.20.30.40 foo" }' #no rulebase given, but has valid tail-field reset_rules add_rule 'rule=:%net:descent::foo' reset_rules 'child' execute '10.20.30.40 foo' assert_output_json_eq '{ "originalmsg": "10.20.30.40 foo", "unparsed-data": "10.20.30.40 foo" }' #empty tail-field given echo empty tail-field given rm tmp.rulebase reset_rules add_rule 'rule=:A%net:descent:./child.rulebase:%' reset_rules 'child' add_rule 'rule=:%ip_addr:ipv4% %tail:rest%' 'child' execute 'A10.20.30.40 foo' assert_output_json_eq '{ "net": { "tail": "foo", "ip_addr": "10.20.30.40" } }' #named tail-field not populated echo tail-field not populated reset_rules add_rule 'rule=:%net:descent:./child.rulebase:foo% foo' reset_rules 'child' add_rule 'rule=:%ip_addr:ipv4% %tail:rest%' 'child' execute '10.20.30.40 foo' assert_output_json_eq '{ "originalmsg": "10.20.30.40 foo", "unparsed-data": "10.20.30.40 foo" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_duration.sh000077500000000000000000000014261511425433100203600ustar00rootroot00000000000000#!/bin/bash # added 2015-03-12 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "duration syntax" add_rule 'version=2' add_rule 'rule=:duration %field:duration% bytes' add_rule 'rule=:duration %field:duration%' execute 'duration 0:00:42 bytes' assert_output_json_eq '{"field": "0:00:42"}' execute 'duration 0:00:42' assert_output_json_eq '{"field": "0:00:42"}' execute 'duration 9:00:42 bytes' assert_output_json_eq '{"field": "9:00:42"}' execute 'duration 00:00:42 bytes' assert_output_json_eq '{"field": "00:00:42"}' execute 'duration 37:59:42 bytes' assert_output_json_eq '{"field": "37:59:42"}' execute 'duration 37:60:42 bytes' assert_output_contains '"unparsed-data": "37:60:42 bytes"' cleanup_tmp_files liblognorm-2.0.8/tests/field_duration_jsoncnf.sh000077500000000000000000000015001511425433100220710ustar00rootroot00000000000000#!/bin/bash # added 2015-03-12 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "duration syntax" add_rule 'version=2' add_rule 'rule=:duration %{"name":"field", "type":"duration"}% bytes' add_rule 'rule=:duration %{"name":"field", "type":"duration"}%' execute 'duration 0:00:42 bytes' assert_output_json_eq '{"field": "0:00:42"}' execute 'duration 0:00:42' assert_output_json_eq '{"field": "0:00:42"}' execute 'duration 9:00:42 bytes' assert_output_json_eq '{"field": "9:00:42"}' execute 'duration 00:00:42 bytes' assert_output_json_eq '{"field": "00:00:42"}' execute 'duration 37:59:42 bytes' assert_output_json_eq '{"field": "37:59:42"}' execute 'duration 37:60:42 bytes' assert_output_contains '"unparsed-data": "37:60:42 bytes"' cleanup_tmp_files liblognorm-2.0.8/tests/field_duration_v1.sh000077500000000000000000000014161511425433100207650ustar00rootroot00000000000000#!/bin/bash # added 2015-03-12 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "duration syntax" add_rule 'rule=:duration %field:duration% bytes' add_rule 'rule=:duration %field:duration%' execute 'duration 0:00:42 bytes' assert_output_json_eq '{"field": "0:00:42"}' execute 'duration 0:00:42' assert_output_json_eq '{"field": "0:00:42"}' execute 'duration 9:00:42 bytes' assert_output_json_eq '{"field": "9:00:42"}' execute 'duration 00:00:42 bytes' assert_output_json_eq '{"field": "00:00:42"}' execute 'duration 37:59:42 bytes' assert_output_json_eq '{"field": "37:59:42"}' execute 'duration 37:60:42 bytes' assert_output_contains '"unparsed-data": "37:60:42 bytes"' cleanup_tmp_files liblognorm-2.0.8/tests/field_float-fmt_number.sh000077500000000000000000000016531511425433100217760ustar00rootroot00000000000000#!/bin/bash # added 2017-10-02 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "float field" add_rule 'version=2' add_rule 'rule=:here is a number %{ "type":"float", "name":"num", "format":"number"}% in floating pt form' execute 'here is a number 15.9 in floating pt form' assert_output_json_eq '{"num": 15.9}' reset_rules # note: floating point numbers are tricky to get right, even more so if negative. add_rule 'version=2' add_rule 'rule=:here is a negative number %{ "type":"float", "name":"num", "format":"number"}% for you' execute 'here is a negative number -4.2 for you' assert_output_json_eq '{"num": -4.2}' reset_rules add_rule 'version=2' add_rule 'rule=:here is another real number %{ "type":"float", "name":"num", "format":"number"}%.' execute 'here is another real number 2.71.' assert_output_json_eq '{"num": 2.71}' cleanup_tmp_files liblognorm-2.0.8/tests/field_float.sh000077500000000000000000000013401511425433100176330ustar00rootroot00000000000000#!/bin/bash # added 2015-02-25 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "float field" add_rule 'version=2' add_rule 'rule=:here is a number %num:float% in floating pt form' execute 'here is a number 15.9 in floating pt form' assert_output_json_eq '{"num": "15.9"}' reset_rules add_rule 'version=2' add_rule 'rule=:here is a negative number %num:float% for you' execute 'here is a negative number -4.2 for you' assert_output_json_eq '{"num": "-4.2"}' reset_rules add_rule 'version=2' add_rule 'rule=:here is another real number %real_no:float%.' execute 'here is another real number 2.71.' assert_output_json_eq '{"real_no": "2.71"}' cleanup_tmp_files liblognorm-2.0.8/tests/field_float_jsoncnf.sh000077500000000000000000000014371511425433100213620ustar00rootroot00000000000000#!/bin/bash # added 2015-02-25 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "float field" add_rule 'version=2' add_rule 'rule=:here is a number %{"name":"num", "type":"float"}% in floating pt form' execute 'here is a number 15.9 in floating pt form' assert_output_json_eq '{"num": "15.9"}' reset_rules add_rule 'version=2' add_rule 'rule=:here is a negative number %{"name":"num", "type":"float"}% for you' execute 'here is a negative number -4.2 for you' assert_output_json_eq '{"num": "-4.2"}' reset_rules add_rule 'version=2' add_rule 'rule=:here is another real number %{"name":"real_no", "type":"float"}%.' execute 'here is another real number 2.71.' assert_output_json_eq '{"real_no": "2.71"}' cleanup_tmp_files liblognorm-2.0.8/tests/field_float_v1.sh000077500000000000000000000012561511425433100202470ustar00rootroot00000000000000#!/bin/bash # added 2015-02-25 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "float field" add_rule 'rule=:here is a number %num:float% in floating pt form' execute 'here is a number 15.9 in floating pt form' assert_output_json_eq '{"num": "15.9"}' reset_rules add_rule 'rule=:here is a negative number %num:float% for you' execute 'here is a negative number -4.2 for you' assert_output_json_eq '{"num": "-4.2"}' reset_rules add_rule 'rule=:here is another real number %real_no:float%.' execute 'here is another real number 2.71.' assert_output_json_eq '{"real_no": "2.71"}' cleanup_tmp_files liblognorm-2.0.8/tests/field_float_with_invalid_ruledef.sh000077500000000000000000000005271511425433100241100ustar00rootroot00000000000000#!/bin/bash # added 2015-02-26 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "float with invalid field-declaration" add_rule 'rule=:%no:flo% foo' execute '10.0 foo' assert_output_json_eq '{ "originalmsg": "10.0 foo", "unparsed-data": "10.0 foo" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_hexnumber-fmt_number.sh000077500000000000000000000011421511425433100226570ustar00rootroot00000000000000#!/bin/bash # added 2017-10-02 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "hexnumber field" add_rule 'version=2' add_rule 'rule=:here is a number %{ "type":"hexnumber", "name":"num", "format":"number"} % in hex form' execute 'here is a number 0x1234 in hex form' assert_output_json_eq '{"num": 4660}' #check cases where parsing failure must occur execute 'here is a number 0x1234in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0x1234in hex form", "unparsed-data": "0x1234in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_hexnumber.sh000077500000000000000000000010741511425433100205270ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "hexnumber field" add_rule 'version=2' add_rule 'rule=:here is a number %num:hexnumber% in hex form' execute 'here is a number 0x1234 in hex form' assert_output_json_eq '{"num": "0x1234"}' #check cases where parsing failure must occur execute 'here is a number 0x1234in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0x1234in hex form", "unparsed-data": "0x1234in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_hexnumber_jsoncnf.sh000077500000000000000000000011161511425433100222440ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "hexnumber field" add_rule 'version=2' add_rule 'rule=:here is a number %{"name":"num", "type":"hexnumber"}% in hex form' execute 'here is a number 0x1234 in hex form' assert_output_json_eq '{"num": "0x1234"}' #check cases where parsing failure must occur execute 'here is a number 0x1234in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0x1234in hex form", "unparsed-data": "0x1234in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_hexnumber_range.sh000077500000000000000000000013731511425433100217050ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "hexnumber field with range checks" add_rule 'version=2' add_rule 'rule=:here is a number %num:hexnumber{"maxval":191}% in hex form' execute 'here is a number 0x12 in hex form' assert_output_json_eq '{"num": "0x12"}' execute 'here is a number 0x0 in hex form' assert_output_json_eq '{"num": "0x0"}' execute 'here is a number 0xBf in hex form' assert_output_json_eq '{"num": "0xBf"}' #check cases where parsing failure must occur execute 'here is a number 0xc0 in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0xc0 in hex form", "unparsed-data": "0xc0 in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_hexnumber_range_jsoncnf.sh000077500000000000000000000014201511425433100234160ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "hexnumber field with range checks" add_rule 'version=2' add_rule 'rule=:here is a number %{"name":"num", "type":"hexnumber", "maxval":191}% in hex form' execute 'here is a number 0x12 in hex form' assert_output_json_eq '{"num": "0x12"}' execute 'here is a number 0x0 in hex form' assert_output_json_eq '{"num": "0x0"}' execute 'here is a number 0xBf in hex form' assert_output_json_eq '{"num": "0xBf"}' #check cases where parsing failure must occur execute 'here is a number 0xc0 in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0xc0 in hex form", "unparsed-data": "0xc0 in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_hexnumber_v1.sh000077500000000000000000000010631511425433100211330ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "hexnumber field" add_rule 'rule=:here is a number %num:hexnumber% in hex form' execute 'here is a number 0x1234 in hex form' assert_output_json_eq '{"num": "0x1234"}' #check cases where parsing failure must occur execute 'here is a number 0x1234in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0x1234in hex form", "unparsed-data": "0x1234in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_interpret.sh000077500000000000000000000045711511425433100205530ustar00rootroot00000000000000#!/bin/bash # added 2014-12-11 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "value interpreting field" add_rule 'rule=:%session_count:interpret:int:word% sessions established' execute '64 sessions established' assert_output_json_eq '{"session_count": 64}' reset_rules add_rule 'rule=:max sessions limit reached: %at_limit:interpret:bool:word%' execute 'max sessions limit reached: true' assert_output_json_eq '{"at_limit": true}' execute 'max sessions limit reached: false' assert_output_json_eq '{"at_limit": false}' execute 'max sessions limit reached: TRUE' assert_output_json_eq '{"at_limit": true}' execute 'max sessions limit reached: FALSE' assert_output_json_eq '{"at_limit": false}' execute 'max sessions limit reached: yes' assert_output_json_eq '{"at_limit": true}' execute 'max sessions limit reached: no' assert_output_json_eq '{"at_limit": false}' execute 'max sessions limit reached: YES' assert_output_json_eq '{"at_limit": true}' execute 'max sessions limit reached: NO' assert_output_json_eq '{"at_limit": false}' reset_rules add_rule 'rule=:record count for shard [%shard:interpret:base16int:char-to:]%] is %record_count:interpret:base10int:number% and %latency_percentile:interpret:float:char-to:\x25%\x25ile latency is %latency:interpret:float:word% %latency_unit:word%' execute 'record count for shard [3F] is 50000 and 99.99%ile latency is 2.1 seconds' assert_output_json_eq '{"shard": 63, "record_count": 50000, "latency_percentile": 99.99, "latency": 2.1, "latency_unit" : "seconds"}' reset_rules add_rule 'rule=:%latency_percentile:interpret:float:char-to:\x25%\x25ile latency is %latency:interpret:float:word%' execute '98.1%ile latency is 1.999123' assert_output_json_eq '{"latency_percentile": 98.1, "latency": 1.999123}' reset_rules add_rule 'rule=:%latency_percentile:interpret:float:number%' add_rule 'rule=:%latency_percentile:interpret:int:number%' add_rule 'rule=:%latency_percentile:interpret:base16int:number%' add_rule 'rule=:%latency_percentile:interpret:base10int:number%' add_rule 'rule=:%latency_percentile:interpret:boolean:number%' execute 'foo' assert_output_json_eq '{ "originalmsg": "foo", "unparsed-data": "foo" }' reset_rules add_rule 'rule=:gc pause: %pause_time:interpret:float:float%ms' execute 'gc pause: 12.3ms' assert_output_json_eq '{"pause_time": 12.3}' cleanup_tmp_files liblognorm-2.0.8/tests/field_interpret_with_invalid_ruledef.sh000077500000000000000000000046751511425433100250270ustar00rootroot00000000000000#!/bin/bash # added 2014-12-11 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "value interpreting field, with invalid ruledef" add_rule 'rule=:%session_count:interpret:int:wd% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' reset_rules add_rule 'rule=:%session_count:interpret:int:% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' reset_rules add_rule 'rule=:%session_count:interpret:int% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' reset_rules add_rule 'rule=:%session_count:interpret:in% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' reset_rules add_rule 'rule=:%session_count:interpret:in:word% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' reset_rules add_rule 'rule=:%session_count:interpret:in:wd% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' reset_rules add_rule 'rule=:%session_count:interpret::word% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' reset_rules add_rule 'rule=:%session_count:interpret::% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' reset_rules add_rule 'rule=:%session_count:inter::% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' reset_rules add_rule 'rule=:%session_count:inter:int:word% sessions established' execute '64 sessions established' assert_output_json_eq '{ "originalmsg": "64 sessions established", "unparsed-data": "64 sessions established" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_ipv6.sh000077500000000000000000000043531511425433100174210ustar00rootroot00000000000000#!/bin/bash # added 2015-06-23 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh add_rule 'version=2' test_def $0 "IPv6 parser" add_rule 'rule=:%f:ipv6%' # examples from RFC4291, sect. 2.2 execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345:6789' assert_output_json_eq '{ "f": "ABCD:EF01:2345:6789:ABCD:EF01:2345:6789" }' execute 'ABCD:EF01:2345:6789:abcd:EF01:2345:6789' # mixed hex case assert_output_json_eq '{ "f": "ABCD:EF01:2345:6789:abcd:EF01:2345:6789" }' execute '2001:DB8:0:0:8:800:200C:417A' assert_output_json_eq '{ "f": "2001:DB8:0:0:8:800:200C:417A" }' execute '0:0:0:0:0:0:0:1' assert_output_json_eq '{ "f": "0:0:0:0:0:0:0:1" }' execute '2001:DB8::8:800:200C:417A' assert_output_json_eq '{ "f": "2001:DB8::8:800:200C:417A" }' execute 'FF01::101' assert_output_json_eq '{ "f": "FF01::101" }' execute '::1' assert_output_json_eq '{ "f": "::1" }' execute '::' assert_output_json_eq '{ "f": "::" }' execute '0:0:0:0:0:0:13.1.68.3' assert_output_json_eq '{ "f": "0:0:0:0:0:0:13.1.68.3" }' execute '::13.1.68.3' assert_output_json_eq '{ "f": "::13.1.68.3" }' execute '::FFFF:129.144.52.38' assert_output_json_eq '{ "f": "::FFFF:129.144.52.38" }' # invalid samples execute '2001:DB8::8::800:200C:417A' # two :: sequences assert_output_json_eq '{ "originalmsg": "2001:DB8::8::800:200C:417A", "unparsed-data": "2001:DB8::8::800:200C:417A" }' execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345::6789' # :: with too many blocks assert_output_json_eq '{ "originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789" }' execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798' # too many blocks (9) assert_output_json_eq '{"originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798" }' execute ':0:0:0:0:0:0:1' # missing first digit assert_output_json_eq '{ "originalmsg": ":0:0:0:0:0:0:1", "unparsed-data": ":0:0:0:0:0:0:1" }' execute '0:0:0:0:0:0:0:' # missing last digit assert_output_json_eq '{ "originalmsg": "0:0:0:0:0:0:0:", "unparsed-data": "0:0:0:0:0:0:0:" }' execute '13.1.68.3' # pure IPv4 address assert_output_json_eq '{ "originalmsg": "13.1.68.3", "unparsed-data": "13.1.68.3" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_ipv6_jsoncnf.sh000077500000000000000000000044001511425433100211320ustar00rootroot00000000000000#!/bin/bash # added 2015-06-23 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "IPv6 parser" add_rule 'version=2' add_rule 'rule=:%{"name":"f", "type":"ipv6"}%' # examples from RFC4291, sect. 2.2 execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345:6789' assert_output_json_eq '{ "f": "ABCD:EF01:2345:6789:ABCD:EF01:2345:6789" }' execute 'ABCD:EF01:2345:6789:abcd:EF01:2345:6789' # mixed hex case assert_output_json_eq '{ "f": "ABCD:EF01:2345:6789:abcd:EF01:2345:6789" }' execute '2001:DB8:0:0:8:800:200C:417A' assert_output_json_eq '{ "f": "2001:DB8:0:0:8:800:200C:417A" }' execute '0:0:0:0:0:0:0:1' assert_output_json_eq '{ "f": "0:0:0:0:0:0:0:1" }' execute '2001:DB8::8:800:200C:417A' assert_output_json_eq '{ "f": "2001:DB8::8:800:200C:417A" }' execute 'FF01::101' assert_output_json_eq '{ "f": "FF01::101" }' execute '::1' assert_output_json_eq '{ "f": "::1" }' execute '::' assert_output_json_eq '{ "f": "::" }' execute '0:0:0:0:0:0:13.1.68.3' assert_output_json_eq '{ "f": "0:0:0:0:0:0:13.1.68.3" }' execute '::13.1.68.3' assert_output_json_eq '{ "f": "::13.1.68.3" }' execute '::FFFF:129.144.52.38' assert_output_json_eq '{ "f": "::FFFF:129.144.52.38" }' # invalid samples execute '2001:DB8::8::800:200C:417A' # two :: sequences assert_output_json_eq '{ "originalmsg": "2001:DB8::8::800:200C:417A", "unparsed-data": "2001:DB8::8::800:200C:417A" }' execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345::6789' # :: with too many blocks assert_output_json_eq '{ "originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789" }' execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798' # too many blocks (9) assert_output_json_eq '{"originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798" }' execute ':0:0:0:0:0:0:1' # missing first digit assert_output_json_eq '{ "originalmsg": ":0:0:0:0:0:0:1", "unparsed-data": ":0:0:0:0:0:0:1" }' execute '0:0:0:0:0:0:0:' # missing last digit assert_output_json_eq '{ "originalmsg": "0:0:0:0:0:0:0:", "unparsed-data": "0:0:0:0:0:0:0:" }' execute '13.1.68.3' # pure IPv4 address assert_output_json_eq '{ "originalmsg": "13.1.68.3", "unparsed-data": "13.1.68.3" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_ipv6_v1.sh000077500000000000000000000043261511425433100200270ustar00rootroot00000000000000#!/bin/bash # added 2015-06-23 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "IPv6 parser" add_rule 'rule=:%f:ipv6%' # examples from RFC4291, sect. 2.2 execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345:6789' assert_output_json_eq '{ "f": "ABCD:EF01:2345:6789:ABCD:EF01:2345:6789" }' execute 'ABCD:EF01:2345:6789:abcd:EF01:2345:6789' # mixed hex case assert_output_json_eq '{ "f": "ABCD:EF01:2345:6789:abcd:EF01:2345:6789" }' execute '2001:DB8:0:0:8:800:200C:417A' assert_output_json_eq '{ "f": "2001:DB8:0:0:8:800:200C:417A" }' execute '0:0:0:0:0:0:0:1' assert_output_json_eq '{ "f": "0:0:0:0:0:0:0:1" }' execute '2001:DB8::8:800:200C:417A' assert_output_json_eq '{ "f": "2001:DB8::8:800:200C:417A" }' execute 'FF01::101' assert_output_json_eq '{ "f": "FF01::101" }' execute '::1' assert_output_json_eq '{ "f": "::1" }' execute '::' assert_output_json_eq '{ "f": "::" }' execute '0:0:0:0:0:0:13.1.68.3' assert_output_json_eq '{ "f": "0:0:0:0:0:0:13.1.68.3" }' execute '::13.1.68.3' assert_output_json_eq '{ "f": "::13.1.68.3" }' execute '::FFFF:129.144.52.38' assert_output_json_eq '{ "f": "::FFFF:129.144.52.38" }' # invalid samples execute '2001:DB8::8::800:200C:417A' # two :: sequences assert_output_json_eq '{ "originalmsg": "2001:DB8::8::800:200C:417A", "unparsed-data": "2001:DB8::8::800:200C:417A" }' execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345::6789' # :: with too many blocks assert_output_json_eq '{ "originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345::6789" }' execute 'ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798' # too many blocks (9) assert_output_json_eq '{"originalmsg": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798", "unparsed-data": "ABCD:EF01:2345:6789:ABCD:EF01:2345:1:6798" }' execute ':0:0:0:0:0:0:1' # missing first digit assert_output_json_eq '{ "originalmsg": ":0:0:0:0:0:0:1", "unparsed-data": ":0:0:0:0:0:0:1" }' execute '0:0:0:0:0:0:0:' # missing last digit assert_output_json_eq '{ "originalmsg": "0:0:0:0:0:0:0:", "unparsed-data": "0:0:0:0:0:0:0:" }' execute '13.1.68.3' # pure IPv4 address assert_output_json_eq '{ "originalmsg": "13.1.68.3", "unparsed-data": "13.1.68.3" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_json.sh000077500000000000000000000040571511425433100175070ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "JSON field" add_rule 'version=2' add_rule 'rule=:%field:json%' execute '{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # let's see if something more complicated still works, so ADD some # more rules add_rule 'rule=:begin %field:json%' add_rule 'rule=:begin %field:json%end' add_rule 'rule=:%field:json%end' execute '{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' #check if trailing whitespace is ignored execute '{"f1": "1", "f2": 2} ' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute 'begin {"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute 'begin {"f1": "1", "f2": 2}end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # note: the parser takes all whitespace after the JSON # to be part of it! execute 'begin {"f1": "1", "f2": 2} end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute 'begin {"f1": "1", "f2": 2} end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '{"f1": "1", "f2": 2}end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' #check cases where parsing failure must occur execute '{"f1": "1", f2: 2}' assert_output_json_eq '{ "originalmsg": "{\"f1\": \"1\", f2: 2}", "unparsed-data": "{\"f1\": \"1\", f2: 2}" }' #some more complex cases add_rule 'rule=:%field1:json%-%field2:json%' execute '{"f1": "1"}-{"f2": 2}' assert_output_json_eq '{ "field2": { "f2": 2 }, "field1": { "f1": "1" } }' # re-check previous def still works execute '{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # now check some strange cases reset_rules add_rule 'version=2' add_rule 'rule=:%field:json%' # this check is because of bug in json-c: # https://github.com/json-c/json-c/issues/181 execute '15:00' assert_output_json_eq '{ "originalmsg": "15:00", "unparsed-data": "15:00" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_json_jsoncnf.sh000077500000000000000000000041041511425433100212200ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "JSON field" add_rule 'version=2' add_rule 'rule=:%{"name":"field", "type":"json"}%' execute '{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # let's see if something more complicated still works, so ADD some # more rules add_rule 'rule=:begin %field:json%' add_rule 'rule=:begin %field:json%end' add_rule 'rule=:%field:json%end' execute '{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' #check if trailing whitespace is ignored execute '{"f1": "1", "f2": 2} ' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute 'begin {"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute 'begin {"f1": "1", "f2": 2}end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # note: the parser takes all whitespace after the JSON # to be part of it! execute 'begin {"f1": "1", "f2": 2} end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute 'begin {"f1": "1", "f2": 2} end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '{"f1": "1", "f2": 2}end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' #check cases where parsing failure must occur execute '{"f1": "1", f2: 2}' assert_output_json_eq '{ "originalmsg": "{\"f1\": \"1\", f2: 2}", "unparsed-data": "{\"f1\": \"1\", f2: 2}" }' #some more complex cases add_rule 'rule=:%field1:json%-%field2:json%' execute '{"f1": "1"}-{"f2": 2}' assert_output_json_eq '{ "field2": { "f2": 2 }, "field1": { "f1": "1" } }' # re-check previous def still works execute '{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # now check some strange cases reset_rules add_rule 'version=2' add_rule 'rule=:%field:json%' # this check is because of bug in json-c: # https://github.com/json-c/json-c/issues/181 execute '15:00' assert_output_json_eq '{ "originalmsg": "15:00", "unparsed-data": "15:00" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_json_v1.sh000077500000000000000000000040051511425433100201060ustar00rootroot00000000000000#!/bin/bash # added 2015-03-01 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "JSON field" add_rule 'rule=:%field:json%' execute '{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # let's see if something more complicated still works, so ADD some # more rules add_rule 'rule=:begin %field:json%' add_rule 'rule=:begin %field:json%end' add_rule 'rule=:%field:json%end' execute '{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' #check if trailing whitespace is ignored execute '{"f1": "1", "f2": 2} ' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute 'begin {"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute 'begin {"f1": "1", "f2": 2}end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # note: the parser takes all whitespace after the JSON # to be part of it! execute 'begin {"f1": "1", "f2": 2} end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute 'begin {"f1": "1", "f2": 2} end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' execute '{"f1": "1", "f2": 2}end' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' #check cases where parsing failure must occur execute '{"f1": "1", f2: 2}' assert_output_json_eq '{ "originalmsg": "{\"f1\": \"1\", f2: 2}", "unparsed-data": "{\"f1\": \"1\", f2: 2}" }' #some more complex cases add_rule 'rule=:%field1:json%-%field2:json%' execute '{"f1": "1"}-{"f2": 2}' assert_output_json_eq '{ "field2": { "f2": 2 }, "field1": { "f1": "1" } }' # re-check previous def still works execute '{"f1": "1", "f2": 2}' assert_output_json_eq '{ "field": { "f1": "1", "f2": 2 } }' # now check some strange cases reset_rules add_rule 'rule=:%field:json%' # this check is because of bug in json-c: # https://github.com/json-c/json-c/issues/181 execute '15:00' assert_output_json_eq '{ "originalmsg": "15:00", "unparsed-data": "15:00" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_kernel_timestamp.sh000077500000000000000000000033371511425433100221010ustar00rootroot00000000000000#!/bin/bash # added 2015-03-12 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "kernel timestamp parser" add_rule 'version=2' add_rule 'rule=:begin %timestamp:kernel-timestamp% end' execute 'begin [12345.123456] end' assert_output_json_eq '{ "timestamp": "[12345.123456]"}' reset_rules add_rule 'version=2' add_rule 'rule=:begin %timestamp:kernel-timestamp%' execute 'begin [12345.123456]' assert_output_json_eq '{ "timestamp": "[12345.123456]"}' reset_rules add_rule 'version=2' add_rule 'rule=:%timestamp:kernel-timestamp%' execute '[12345.123456]' assert_output_json_eq '{ "timestamp": "[12345.123456]"}' execute '[154469.133028]' assert_output_json_eq '{ "timestamp": "[154469.133028]"}' execute '[123456789012.123456]' assert_output_json_eq '{ "timestamp": "[123456789012.123456]"}' #check cases where parsing failure must occur execute '[1234.123456]' assert_output_json_eq '{"originalmsg": "[1234.123456]", "unparsed-data": "[1234.123456]" }' execute '[1234567890123.123456]' assert_output_json_eq '{"originalmsg": "[1234567890123.123456]", "unparsed-data": "[1234567890123.123456]" }' execute '[123456789012.12345]' assert_output_json_eq '{ "originalmsg": "[123456789012.12345]", "unparsed-data": "[123456789012.12345]" }' execute '[123456789012.1234567]' assert_output_json_eq '{ "originalmsg": "[123456789012.1234567]", "unparsed-data": "[123456789012.1234567]" }' execute '(123456789012.123456]' assert_output_json_eq '{ "originalmsg": "(123456789012.123456]", "unparsed-data": "(123456789012.123456]" }' execute '[123456789012.123456' assert_output_json_eq '{ "originalmsg": "[123456789012.123456", "unparsed-data": "[123456789012.123456" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_kernel_timestamp_jsoncnf.sh000077500000000000000000000034361511425433100236210ustar00rootroot00000000000000#!/bin/bash # added 2015-03-12 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "kernel timestamp parser" add_rule 'version=2' add_rule 'rule=:begin %{"name":"timestamp", "type":"kernel-timestamp"}% end' execute 'begin [12345.123456] end' assert_output_json_eq '{ "timestamp": "[12345.123456]"}' reset_rules add_rule 'version=2' add_rule 'rule=:begin %{"name":"timestamp", "type":"kernel-timestamp"}%' execute 'begin [12345.123456]' assert_output_json_eq '{ "timestamp": "[12345.123456]"}' reset_rules add_rule 'version=2' add_rule 'rule=:%{"name":"timestamp", "type":"kernel-timestamp"}%' execute '[12345.123456]' assert_output_json_eq '{ "timestamp": "[12345.123456]"}' execute '[154469.133028]' assert_output_json_eq '{ "timestamp": "[154469.133028]"}' execute '[123456789012.123456]' assert_output_json_eq '{ "timestamp": "[123456789012.123456]"}' #check cases where parsing failure must occur execute '[1234.123456]' assert_output_json_eq '{"originalmsg": "[1234.123456]", "unparsed-data": "[1234.123456]" }' execute '[1234567890123.123456]' assert_output_json_eq '{"originalmsg": "[1234567890123.123456]", "unparsed-data": "[1234567890123.123456]" }' execute '[123456789012.12345]' assert_output_json_eq '{ "originalmsg": "[123456789012.12345]", "unparsed-data": "[123456789012.12345]" }' execute '[123456789012.1234567]' assert_output_json_eq '{ "originalmsg": "[123456789012.1234567]", "unparsed-data": "[123456789012.1234567]" }' execute '(123456789012.123456]' assert_output_json_eq '{ "originalmsg": "(123456789012.123456]", "unparsed-data": "(123456789012.123456]" }' execute '[123456789012.123456' assert_output_json_eq '{ "originalmsg": "[123456789012.123456", "unparsed-data": "[123456789012.123456" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_kernel_timestamp_v1.sh000077500000000000000000000032551511425433100225060ustar00rootroot00000000000000#!/bin/bash # added 2015-03-12 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "kernel timestamp parser" add_rule 'rule=:begin %timestamp:kernel-timestamp% end' execute 'begin [12345.123456] end' assert_output_json_eq '{ "timestamp": "[12345.123456]"}' reset_rules add_rule 'rule=:begin %timestamp:kernel-timestamp%' execute 'begin [12345.123456]' assert_output_json_eq '{ "timestamp": "[12345.123456]"}' reset_rules add_rule 'rule=:%timestamp:kernel-timestamp%' execute '[12345.123456]' assert_output_json_eq '{ "timestamp": "[12345.123456]"}' execute '[154469.133028]' assert_output_json_eq '{ "timestamp": "[154469.133028]"}' execute '[123456789012.123456]' assert_output_json_eq '{ "timestamp": "[123456789012.123456]"}' #check cases where parsing failure must occur execute '[1234.123456]' assert_output_json_eq '{"originalmsg": "[1234.123456]", "unparsed-data": "[1234.123456]" }' execute '[1234567890123.123456]' assert_output_json_eq '{"originalmsg": "[1234567890123.123456]", "unparsed-data": "[1234567890123.123456]" }' execute '[123456789012.12345]' assert_output_json_eq '{ "originalmsg": "[123456789012.12345]", "unparsed-data": "[123456789012.12345]" }' execute '[123456789012.1234567]' assert_output_json_eq '{ "originalmsg": "[123456789012.1234567]", "unparsed-data": "[123456789012.1234567]" }' execute '(123456789012.123456]' assert_output_json_eq '{ "originalmsg": "(123456789012.123456]", "unparsed-data": "(123456789012.123456]" }' execute '[123456789012.123456' assert_output_json_eq '{ "originalmsg": "[123456789012.123456", "unparsed-data": "[123456789012.123456" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_mac48.sh000077500000000000000000000013111511425433100174400ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "dmac48 syntax" reset_rules add_rule 'version=2' add_rule 'rule=:%field:mac48%' execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"field": "f0:f6:1c:5f:cc:a2"}' execute 'f0-f6-1c-5f-cc-a2' assert_output_json_eq '{"field": "f0-f6-1c-5f-cc-a2"}' # things that need to NOT match execute 'f0-f6:1c:5f:cc-a2' assert_output_json_eq '{ "originalmsg": "f0-f6:1c:5f:cc-a2", "unparsed-data": "f0-f6:1c:5f:cc-a2" }' execute 'f0:f6:1c:xf:cc:a2' assert_output_json_eq '{ "originalmsg": "f0:f6:1c:xf:cc:a2", "unparsed-data": "f0:f6:1c:xf:cc:a2" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_mac48_jsoncnf.sh000077500000000000000000000013221511425433100211620ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "dmac48 syntax" add_rule 'version=2' add_rule 'rule=:%{"name":"field", "type":"mac48"}%' execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"field": "f0:f6:1c:5f:cc:a2"}' execute 'f0-f6-1c-5f-cc-a2' assert_output_json_eq '{"field": "f0-f6-1c-5f-cc-a2"}' # things that need to NOT match execute 'f0-f6:1c:5f:cc-a2' assert_output_json_eq '{ "originalmsg": "f0-f6:1c:5f:cc-a2", "unparsed-data": "f0-f6:1c:5f:cc-a2" }' execute 'f0:f6:1c:xf:cc:a2' assert_output_json_eq '{ "originalmsg": "f0:f6:1c:xf:cc:a2", "unparsed-data": "f0:f6:1c:xf:cc:a2" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_mac48_v1.sh000077500000000000000000000012501511425433100200500ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "dmac48 syntax" add_rule 'rule=:%field:mac48%' execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"field": "f0:f6:1c:5f:cc:a2"}' execute 'f0-f6-1c-5f-cc-a2' assert_output_json_eq '{"field": "f0-f6-1c-5f-cc-a2"}' # things that need to NOT match execute 'f0-f6:1c:5f:cc-a2' assert_output_json_eq '{ "originalmsg": "f0-f6:1c:5f:cc-a2", "unparsed-data": "f0-f6:1c:5f:cc-a2" }' execute 'f0:f6:1c:xf:cc:a2' assert_output_json_eq '{ "originalmsg": "f0:f6:1c:xf:cc:a2", "unparsed-data": "f0:f6:1c:xf:cc:a2" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_name_value.sh000077500000000000000000000024251511425433100206470ustar00rootroot00000000000000#!/bin/bash # added 2015-04-25 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "name/value parser" add_rule 'version=2' add_rule 'rule=:%f:name-value-list%' execute 'name=value' assert_output_json_eq '{ "f": { "name": "value" } }' execute 'name1=value1 name2=value2 name3=value3' assert_output_json_eq '{ "f": { "name1": "value1", "name2": "value2", "name3": "value3" } }' execute 'name1=value1 name2=value2 name3=value3 ' assert_output_json_eq '{ "f": { "name1": "value1", "name2": "value2", "name3": "value3" } }' execute 'name1= name2=value2 name3=value3 ' assert_output_json_eq '{ "f": { "name1": "", "name2": "value2", "name3": "value3" } }' execute 'origin=core.action processed=67 failed=0 suspended=0 suspended.duration=0 resumed=0 ' assert_output_json_eq '{ "f": { "origin": "core.action", "processed": "67", "failed": "0", "suspended": "0", "suspended.duration": "0", "resumed": "0" } }' # check for required non-matches execute 'name' assert_output_json_eq ' {"originalmsg": "name", "unparsed-data": "name" }' execute 'noname1 name2=value2 name3=value3 ' assert_output_json_eq '{ "originalmsg": "noname1 name2=value2 name3=value3 ", "unparsed-data": "noname1 name2=value2 name3=value3 " }' cleanup_tmp_files liblognorm-2.0.8/tests/field_name_value_jsoncnf.sh000077500000000000000000000024521511425433100223670ustar00rootroot00000000000000#!/bin/bash # added 2015-04-25 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "name/value parser" add_rule 'version=2' add_rule 'rule=:%{"name":"f", "type":"name-value-list"}%' execute 'name=value' assert_output_json_eq '{ "f": { "name": "value" } }' execute 'name1=value1 name2=value2 name3=value3' assert_output_json_eq '{ "f": { "name1": "value1", "name2": "value2", "name3": "value3" } }' execute 'name1=value1 name2=value2 name3=value3 ' assert_output_json_eq '{ "f": { "name1": "value1", "name2": "value2", "name3": "value3" } }' execute 'name1= name2=value2 name3=value3 ' assert_output_json_eq '{ "f": { "name1": "", "name2": "value2", "name3": "value3" } }' execute 'origin=core.action processed=67 failed=0 suspended=0 suspended.duration=0 resumed=0 ' assert_output_json_eq '{ "f": { "origin": "core.action", "processed": "67", "failed": "0", "suspended": "0", "suspended.duration": "0", "resumed": "0" } }' # check for required non-matches execute 'name' assert_output_json_eq ' {"originalmsg": "name", "unparsed-data": "name" }' execute 'noname1 name2=value2 name3=value3 ' assert_output_json_eq '{ "originalmsg": "noname1 name2=value2 name3=value3 ", "unparsed-data": "noname1 name2=value2 name3=value3 " }' cleanup_tmp_files liblognorm-2.0.8/tests/field_name_value_quoted.sh000077500000000000000000000034301511425433100222250ustar00rootroot00000000000000#!/bin/bash # added 2021-11-08 by @KGuillemot # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "name/value parser" add_rule 'version=2' add_rule 'rule=:%f:name-value-list%' execute 'name="value"' assert_output_json_eq '{ "f": { "name": "value" } }' execute 'name1="value1" name2="value2" name3="value3"' assert_output_json_eq '{ "f": { "name1": "value1", "name2": "value2", "name3": "value3" } }' execute 'name1="value1 name2=value2" name3=value3 ' assert_output_json_eq '{ "f": { "name1": "value1 name2=value2", "name3": "value3" } }' execute 'name1="" name2="value2" name3="value3" ' assert_output_json_eq '{ "f": { "name1": "", "name2": "value2", "name3": "value3" } }' execute 'origin="core.action" processed=67 failed=0 suspended=0 suspended.duration=0 resumed=0 ' assert_output_json_eq '{ "f": { "origin": "core.action", "processed": "67", "failed": "0", "suspended": "0", "suspended.duration": "0", "resumed": "0" } }' # check escaped caracters execute 'name1="a\"b" name2="c\\\"d" name3="e\\\\\"f" ' assert_output_json_eq '{ "f": { "name1": "a\"b", "name2": "c\\\"d", "name3": "e\\\\\"f" } }' execute 'name1="a\"b\\" name2="c\\\"d\\\\" name3="e\\\\\"f\\\\\\" ' assert_output_json_eq '{ "f": { "name1": "a\"b\\", "name2": "c\\\"d\\\\", "name3": "e\\\\\"f\\\\\\" } }' # check for required non-matches execute 'name' assert_output_json_eq ' {"originalmsg": "name", "unparsed-data": "name" }' # check escaped caracters execute 'name1="" rest' assert_output_json_eq ' {"originalmsg": "name1=\"\" rest", "unparsed-data": "rest" }' execute 'noname1 name2="value2" name3="value3" ' assert_output_json_eq '{ "originalmsg": "noname1 name2=\"value2\" name3=\"value3\" ", "unparsed-data": "noname1 name2=\"value2\" name3=\"value3\" " }' cleanup_tmp_files liblognorm-2.0.8/tests/field_name_value_v1.sh000077500000000000000000000024001511425433100212460ustar00rootroot00000000000000#!/bin/bash # added 2015-04-25 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "name/value parser" add_rule 'rule=:%f:name-value-list%' execute 'name=value' assert_output_json_eq '{ "f": { "name": "value" } }' execute 'name1=value1 name2=value2 name3=value3' assert_output_json_eq '{ "f": { "name1": "value1", "name2": "value2", "name3": "value3" } }' execute 'name1=value1 name2=value2 name3=value3 ' assert_output_json_eq '{ "f": { "name1": "value1", "name2": "value2", "name3": "value3" } }' execute 'name1= name2=value2 name3=value3 ' assert_output_json_eq '{ "f": { "name1": "", "name2": "value2", "name3": "value3" } }' execute 'origin=core.action processed=67 failed=0 suspended=0 suspended.duration=0 resumed=0 ' assert_output_json_eq '{ "f": { "origin": "core.action", "processed": "67", "failed": "0", "suspended": "0", "suspended.duration": "0", "resumed": "0" } }' # check for required non-matches execute 'name' assert_output_json_eq ' {"originalmsg": "name", "unparsed-data": "name" }' execute 'noname1 name2=value2 name3=value3 ' assert_output_json_eq '{ "originalmsg": "noname1 name2=value2 name3=value3 ", "unparsed-data": "noname1 name2=value2 name3=value3 " }' cleanup_tmp_files liblognorm-2.0.8/tests/field_number-fmt_number.sh000077500000000000000000000011371511425433100221560ustar00rootroot00000000000000#!/bin/bash # added 2017-10-02 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "number field in native format" add_rule 'version=2' add_rule 'rule=:here is a number %{ "type":"number", "name":"num", "format":"number"}% in dec form' execute 'here is a number 1234 in dec form' assert_output_json_eq '{"num": 1234}' #check cases where parsing failure must occur execute 'here is a number 1234in dec form' assert_output_json_eq '{ "originalmsg": "here is a number 1234in dec form", "unparsed-data": "in dec form" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_number.sh000077500000000000000000000010471511425433100200220ustar00rootroot00000000000000#!/bin/bash # added 2017-10-02 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "number field" add_rule 'version=2' add_rule 'rule=:here is a number %num:number% in dec form' execute 'here is a number 1234 in dec form' assert_output_json_eq '{"num": "1234"}' #check cases where parsing failure must occur execute 'here is a number 1234in dec form' assert_output_json_eq '{ "originalmsg": "here is a number 1234in dec form", "unparsed-data": "in dec form" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_number_maxval.sh000077500000000000000000000011321511425433100213650ustar00rootroot00000000000000#!/bin/bash # added 2017-10-02 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "number field with maxval" add_rule 'version=2' add_rule 'rule=:here is a number %{ "type":"number", "name":"num", "maxval":1000}% in dec form' execute 'here is a number 234 in dec form' assert_output_json_eq '{"num": "234"}' #check cases where parsing failure must occur execute 'here is a number 1234in dec form' assert_output_json_eq '{ "originalmsg": "here is a number 1234in dec form", "unparsed-data": "1234in dec form" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_recursive.sh000077500000000000000000000056771511425433100205560ustar00rootroot00000000000000#!/bin/bash # added 2014-11-26 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "recursive parsing field" #tail recursion with default tail field add_rule 'rule=:%word:word% %next:recursive%' add_rule 'rule=:%word:word%' execute '123 abc 456 def' assert_output_json_eq '{"word": "123", "next": {"word": "abc", "next": {"word": "456", "next" : {"word": "def"}}}}' #tail recursion with explicitly named 'tail' field reset_rules add_rule 'rule=:%word:word% %next:recursive:tail%' add_rule 'rule=:%word:word%' execute '123 abc 456 def' assert_output_json_eq '{"word": "123", "next": {"word": "abc", "next": {"word": "456", "next" : {"word": "def"}}}}' #tail recursion with tail field having arbitrary name reset_rules add_rule 'rule=:%word:word% %next:recursive:foo%' add_rule 'rule=:%word:word%' execute '123 abc 456 def' assert_output_json_eq '{"word": "123", "next": {"word": "abc", "next": {"word": "456", "next" : {"word": "def"}}}}' #non tail recursion with default tail field reset_rules add_rule 'rule=:blocked on %device:word% %net:recursive%at %tm:date-rfc5424%' add_rule 'rule=:%ip_addr:ipv4% %tail:rest%' add_rule 'rule=:%subnet_addr:ipv4%/%mask:number% %tail:rest%' execute 'blocked on gw-1 10.20.30.40 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"ip_addr": "10.20.30.40"}, "tm": "2014-12-08T08:53:33.05+05:30"}' execute 'blocked on gw-1 10.20.30.40/16 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"subnet_addr": "10.20.30.40", "mask": "16"}, "tm": "2014-12-08T08:53:33.05+05:30"}' #non tail recursion with tail field being explicitly named 'tail' reset_rules add_rule 'rule=:blocked on %device:word% %net:recursive:tail%at %tm:date-rfc5424%' add_rule 'rule=:%ip_addr:ipv4% %tail:rest%' add_rule 'rule=:%subnet_addr:ipv4%/%mask:number% %tail:rest%' execute 'blocked on gw-1 10.20.30.40 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"ip_addr": "10.20.30.40"}, "tm": "2014-12-08T08:53:33.05+05:30"}' execute 'blocked on gw-1 10.20.30.40/16 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"subnet_addr": "10.20.30.40", "mask": "16"}, "tm": "2014-12-08T08:53:33.05+05:30"}' #non tail recursion with tail field having arbitrary name reset_rules add_rule 'rule=:blocked on %device:word% %net:recursive:remaining%at %tm:date-rfc5424%' add_rule 'rule=:%ip_addr:ipv4% %remaining:rest%' add_rule 'rule=:%subnet_addr:ipv4%/%mask:number% %remaining:rest%' execute 'blocked on gw-1 10.20.30.40 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"ip_addr": "10.20.30.40"}, "tm": "2014-12-08T08:53:33.05+05:30"}' execute 'blocked on gw-1 10.20.30.40/16 at 2014-12-08T08:53:33.05+05:30' assert_output_json_eq '{"device": "gw-1", "net": {"subnet_addr": "10.20.30.40", "mask": "16"}, "tm": "2014-12-08T08:53:33.05+05:30"}' cleanup_tmp_files liblognorm-2.0.8/tests/field_regex_default_group_parse_and_return.sh000077500000000000000000000006551511425433100262030ustar00rootroot00000000000000#!/bin/bash # added 2014-11-14 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 export ln_opts='-oallowRegex' test_def $0 "type ERE for regex field" add_rule 'rule=:%first:regex:[a-z]+% %second:regex:\d+\x25\x3a[a-f0-9]+\x25%' execute 'foo 122%:7a%' assert_output_contains '"first": "foo"' assert_output_contains '"second": "122%:7a%"' cleanup_tmp_files liblognorm-2.0.8/tests/field_regex_invalid_args.sh000077500000000000000000000023051511425433100223640ustar00rootroot00000000000000#!/bin/bash # added 2014-11-14 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 export ln_opts='-oallowRegex' . $srcdir/exec.sh test_def $0 "invalid type for regex field with everything else defaulted" add_rule 'rule=:%first:regex:[a-z]+:Q%' execute 'foo' assert_output_contains '"originalmsg": "foo"' assert_output_contains '"unparsed-data": "foo"' reset_rules add_rule 'rule=:%first:regex:[a-z]+:%' execute 'foo' assert_output_contains '"originalmsg": "foo"' assert_output_contains '"unparsed-data": "foo"' reset_rules add_rule 'rule=:%first:regex:[a-z]+:0:%' execute 'foo' assert_output_contains '"originalmsg": "foo"' assert_output_contains '"unparsed-data": "foo"' reset_rules add_rule 'rule=:%first:regex:[a-z]+:0:0q%' execute 'foo' assert_output_contains '"originalmsg": "foo"' assert_output_contains '"unparsed-data": "foo"' reset_rules add_rule 'rule=:%first:regex:[a-z]+:0a:0%' execute 'foo' assert_output_contains '"originalmsg": "foo"' assert_output_contains '"unparsed-data": "foo"' reset_rules add_rule 'rule=:%first:regex:::::%%%' execute 'foo' assert_output_contains '"originalmsg": "foo"' assert_output_contains '"unparsed-data": "foo"' cleanup_tmp_files liblognorm-2.0.8/tests/field_regex_while_regex_support_is_disabled.sh000077500000000000000000000005541511425433100263460ustar00rootroot00000000000000#!/bin/bash # added 2014-11-14 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "field regex, while regex support is disabled" add_rule 'rule=:%first:regex:[a-z]+%' execute 'foo' assert_output_contains '"originalmsg": "foo"' assert_output_contains '"unparsed-data": "foo"' cleanup_tmp_files liblognorm-2.0.8/tests/field_regex_with_consume_group.sh000077500000000000000000000012461511425433100236450ustar00rootroot00000000000000#!/bin/bash # added 2014-11-14 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 export ln_opts='-oallowRegex' test_def $0 "regex field with consume-group" add_rule 'rule=:%first:regex:([a-z]{2}([a-f0-9]+,)+):0%%rest:rest%' execute 'ad1234abcd,4567ef12,8901abef' assert_output_contains '"first": "ad1234abcd,4567ef12,"' assert_output_contains '"rest": "8901abef"' reset_rules add_rule 'rule=:%first:regex:(([a-z]{2})([a-f0-9]+,)+):2%%rest:rest%' execute 'ad1234abcd,4567ef12,8901abef' assert_output_contains '"first": "ad"' assert_output_contains '"rest": "1234abcd,4567ef12,8901abef"' cleanup_tmp_files liblognorm-2.0.8/tests/field_regex_with_consume_group_and_return_group.sh000077500000000000000000000012761511425433100273050ustar00rootroot00000000000000#!/bin/bash # added 2014-11-14 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh export ln_opts='-oallowRegex' no_solaris10 test_def $0 "regex field with consume-group and return-group" set -x add_rule 'rule=:%first:regex:[a-z]{2}(([a-f0-9]+),)+:0:2%%rest:rest%' execute 'ad1234abcd,4567ef12,8901abef' assert_output_contains '"first": "4567ef12"' assert_output_contains '"rest": "8901abef"' reset_rules add_rule 'rule=:%first:regex:(([a-z]{2})(([a-f0-9]+),)+):2:4%%rest:rest%' execute 'ad1234abcd,4567ef12,8901abef' assert_output_contains '"first": "4567ef12"' assert_output_contains '"rest": "1234abcd,4567ef12,8901abef"' cleanup_tmp_files liblognorm-2.0.8/tests/field_regex_with_negation.sh000077500000000000000000000011741511425433100225640ustar00rootroot00000000000000#!/bin/bash # added 2014-11-17 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 export ln_opts='-oallowRegex' test_def $0 "regex field with negation" add_rule 'rule=:%text:regex:[^,]+%,%more:rest%' execute '123,abc' assert_output_contains '"text": "123"' assert_output_contains '"more": "abc"' reset_rules add_rule 'rule=:%text:regex:([^ ,|]+( |\||,)?)+%%more:rest%' execute '123 abc|456 789,def|ghi,jkl| and some more text' assert_output_contains '"text": "123 abc|456 789,def|ghi,jkl|"' assert_output_contains '"more": " and some more text"' cleanup_tmp_files liblognorm-2.0.8/tests/field_rest.sh000077500000000000000000000035741511425433100175160ustar00rootroot00000000000000#!/bin/bash # some more tests for the "rest" motif, especially to ensure that # "rest" will not interfere with more specific rules. # added 2015-04-27 # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "rest matches" #tail recursion with default tail field add_rule 'version=2' add_rule 'rule=:%iface:char-to:\x3a%\x3a%ip:ipv4%/%port:number% (%label2:char-to:)%)' add_rule 'rule=:%iface:char-to:\x3a%\x3a%ip:ipv4%/%port:number% (%label2:char-to:)%)%tail:rest%' add_rule 'rule=:%iface:char-to:\x3a%\x3a%ip:ipv4%/%port:number%' add_rule 'rule=:%iface:char-to:\x3a%\x3a%ip:ipv4%/%port:number%%tail:rest%' # real-world cisco samples execute 'Outside:10.20.30.40/35 (40.30.20.10/35)' assert_output_json_eq '{ "label2": "40.30.20.10\/35", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' execute 'Outside:10.20.30.40/35 (40.30.20.10/35) with rest' assert_output_json_eq '{ "tail": " with rest", "label2": "40.30.20.10\/35", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' execute 'Outside:10.20.30.40/35 (40.30.20.10/35 brace missing' assert_output_json_eq '{ "tail": " (40.30.20.10\/35 brace missing", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' execute 'Outside:10.20.30.40/35 40.30.20.10/35' assert_output_json_eq '{ "tail": " 40.30.20.10\/35", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' # # test expected mismatches # execute 'not at all!' assert_output_json_eq '{ "originalmsg": "not at all!", "unparsed-data": "not at all!" }' execute 'Outside 10.20.30.40/35 40.30.20.10/35' assert_output_json_eq '{ "originalmsg": "Outside 10.20.30.40\/35 40.30.20.10\/35", "unparsed-data": "Outside 10.20.30.40\/35 40.30.20.10\/35" }' execute 'Outside:10.20.30.40/aa 40.30.20.10/35' assert_output_json_eq '{ "originalmsg": "Outside:10.20.30.40\/aa 40.30.20.10\/35", "unparsed-data": "aa 40.30.20.10\/35" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_rest_jsoncnf.sh000077500000000000000000000044161511425433100212320ustar00rootroot00000000000000#!/bin/bash # some more tests for the "rest" motif, especially to ensure that # "rest" will not interfere with more specific rules. # added 2015-04-27 # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "rest matches" #tail recursion with default tail field add_rule 'version=2' add_rule 'rule=:%{"name":"iface", "type":"char-to", "extradata":":"}%:%{"name":"ip", "type":"ipv4"}%/%{"name":"port", "type":"number"}% (%{"name":"label2", "type":"char-to", "extradata":")"}%)' add_rule 'rule=:%{"name":"iface", "type":"char-to", "extradata":":"}%:%{"name":"ip", "type":"ipv4"}%/%{"name":"port", "type":"number"}% (%{"name":"label2", "type":"char-to", "extradata":")"}%)%{"name":"tail", "type":"rest"}%' add_rule 'rule=:%{"name":"iface", "type":"char-to", "extradata":":"}%:%{"name":"ip", "type":"ipv4"}%/%{"name":"port", "type":"number"}%' add_rule 'rule=:%{"name":"iface", "type":"char-to", "extradata":":"}%:%{"name":"ip", "type":"ipv4"}%/%{"name":"port", "type":"number"}%%{"name":"tail", "type":"rest"}%' # real-world cisco samples execute 'Outside:10.20.30.40/35 (40.30.20.10/35)' assert_output_json_eq '{ "label2": "40.30.20.10\/35", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' execute 'Outside:10.20.30.40/35 (40.30.20.10/35) with rest' assert_output_json_eq '{ "tail": " with rest", "label2": "40.30.20.10\/35", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' execute 'Outside:10.20.30.40/35 (40.30.20.10/35 brace missing' assert_output_json_eq '{ "tail": " (40.30.20.10\/35 brace missing", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' execute 'Outside:10.20.30.40/35 40.30.20.10/35' assert_output_json_eq '{ "tail": " 40.30.20.10\/35", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' # # test expected mismatches # execute 'not at all!' assert_output_json_eq '{ "originalmsg": "not at all!", "unparsed-data": "not at all!" }' execute 'Outside 10.20.30.40/35 40.30.20.10/35' assert_output_json_eq '{ "originalmsg": "Outside 10.20.30.40\/35 40.30.20.10\/35", "unparsed-data": "Outside 10.20.30.40\/35 40.30.20.10\/35" }' execute 'Outside:10.20.30.40/aa 40.30.20.10/35' assert_output_json_eq '{ "originalmsg": "Outside:10.20.30.40\/aa 40.30.20.10\/35", "unparsed-data": "aa 40.30.20.10\/35" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_rest_v1.sh000077500000000000000000000035641511425433100201230ustar00rootroot00000000000000#!/bin/bash # some more tests for the "rest" motif, especially to ensure that # "rest" will not interfere with more specific rules. # added 2015-04-27 # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "rest matches" #tail recursion with default tail field add_rule 'rule=:%iface:char-to:\x3a%\x3a%ip:ipv4%/%port:number% (%label2:char-to:)%)' add_rule 'rule=:%iface:char-to:\x3a%\x3a%ip:ipv4%/%port:number% (%label2:char-to:)%)%tail:rest%' add_rule 'rule=:%iface:char-to:\x3a%\x3a%ip:ipv4%/%port:number%' add_rule 'rule=:%iface:char-to:\x3a%\x3a%ip:ipv4%/%port:number%%tail:rest%' # real-world cisco samples execute 'Outside:10.20.30.40/35 (40.30.20.10/35)' assert_output_json_eq '{ "label2": "40.30.20.10\/35", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' execute 'Outside:10.20.30.40/35 (40.30.20.10/35) with rest' assert_output_json_eq '{ "tail": " with rest", "label2": "40.30.20.10\/35", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' execute 'Outside:10.20.30.40/35 (40.30.20.10/35 brace missing' assert_output_json_eq '{ "tail": " (40.30.20.10\/35 brace missing", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' execute 'Outside:10.20.30.40/35 40.30.20.10/35' assert_output_json_eq '{ "tail": " 40.30.20.10\/35", "port": "35", "ip": "10.20.30.40", "iface": "Outside" }' # # test expected mismatches # execute 'not at all!' assert_output_json_eq '{ "originalmsg": "not at all!", "unparsed-data": "not at all!" }' execute 'Outside 10.20.30.40/35 40.30.20.10/35' assert_output_json_eq '{ "originalmsg": "Outside 10.20.30.40\/35 40.30.20.10\/35", "unparsed-data": "Outside 10.20.30.40\/35 40.30.20.10\/35" }' execute 'Outside:10.20.30.40/aa 40.30.20.10/35' assert_output_json_eq '{ "originalmsg": "Outside:10.20.30.40\/aa 40.30.20.10\/35", "unparsed-data": "aa 40.30.20.10\/35" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_rfc5424timestamp-fmt_timestamp-unix-ms.sh000077500000000000000000000027651511425433100260240ustar00rootroot00000000000000#!/bin/bash # added 2017-10-02 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "RFC5424 timestamp in timestamp-unix format" add_rule 'version=2' add_rule 'rule=:here is a timestamp %{ "type":"date-rfc5424", "name":"num", "format":"timestamp-unix-ms"}% in RFC5424 format' execute 'here is a timestamp 2000-03-11T14:15:16+01:00 in RFC5424 format' assert_output_json_eq '{ "num": 952780516000}' # with milliseconds (too-low precision) execute 'here is a timestamp 2000-03-11T14:15:16.1+01:00 in RFC5424 format' assert_output_json_eq '{ "num": 952780516100 }' execute 'here is a timestamp 2000-03-11T14:15:16.12+01:00 in RFC5424 format' assert_output_json_eq '{ "num": 952780516120 }' # with milliseconds (exactly right precision) execute 'here is a timestamp 2000-03-11T14:15:16.123+01:00 in RFC5424 format' assert_output_json_eq '{ "num": 952780516123 }' # with overdone precision execute 'here is a timestamp 2000-03-11T14:15:16.1234+01:00 in RFC5424 format' assert_output_json_eq '{ "num": 952780516123 }' execute 'here is a timestamp 2000-03-11T14:15:16.123456789+01:00 in RFC5424 format' assert_output_json_eq '{ "num": 952780516123 }' #check cases where parsing failure must occur execute 'here is a timestamp 2000-03-11T14:15:16+01:00in RFC5424 format' assert_output_json_eq '{ "originalmsg": "here is a timestamp 2000-03-11T14:15:16+01:00in RFC5424 format", "unparsed-data": "2000-03-11T14:15:16+01:00in RFC5424 format" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_rfc5424timestamp-fmt_timestamp-unix.sh000077500000000000000000000016631511425433100254030ustar00rootroot00000000000000#!/bin/bash # added 2017-10-02 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "RFC5424 timestamp in timestamp-unix format" add_rule 'version=2' add_rule 'rule=:here is a timestamp %{ "type":"date-rfc5424", "name":"num", "format":"timestamp-unix"}% in RFC5424 format' execute 'here is a timestamp 2000-03-11T14:15:16+01:00 in RFC5424 format' assert_output_json_eq '{"num": 952780516}' # with milliseconds (must be ignored with this format!) execute 'here is a timestamp 2000-03-11T14:15:16.321+01:00 in RFC5424 format' assert_output_json_eq '{"num": 952780516}' #check cases where parsing failure must occur execute 'here is a timestamp 2000-03-11T14:15:16+01:00in RFC5424 format' assert_output_json_eq '{ "originalmsg": "here is a timestamp 2000-03-11T14:15:16+01:00in RFC5424 format", "unparsed-data": "2000-03-11T14:15:16+01:00in RFC5424 format" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_string.sh000077500000000000000000000024031511425433100200350ustar00rootroot00000000000000#!/bin/bash # added 2015-09-02 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "string syntax" reset_rules add_rule 'version=2' add_rule 'rule=:a %f:string% b' execute 'a test b' assert_output_json_eq '{"f": "test"}' execute 'a "test" b' assert_output_json_eq '{"f": "test"}' execute 'a "test with space" b' assert_output_json_eq '{"f": "test with space"}' execute 'a "test with "" double escape" b' assert_output_json_eq '{ "f": "test with \" double escape" }' execute 'a "test with \" backslash escape" b' assert_output_json_eq '{ "f": "test with \" backslash escape" }' echo test quoting.mode reset_rules add_rule 'version=2' add_rule 'rule=:a %f:string{"quoting.mode":"none"}% b' execute 'a test b' assert_output_json_eq '{"f": "test"}' execute 'a "test" b' assert_output_json_eq '{"f": "\"test\""}' echo "test quoting.char.*" reset_rules add_rule 'version=2' add_rule 'rule=:a %f:string{"quoting.char.begin":"[", "quoting.char.end":"]"}% b' execute 'a test b' assert_output_json_eq '{"f": "test"}' execute 'a [test] b' assert_output_json_eq '{"f": "test"}' execute 'a [test test2] b' assert_output_json_eq '{"f": "test test2"}' # things that need to NOT match cleanup_tmp_files liblognorm-2.0.8/tests/field_string_doc_sample_lazy.sh000077500000000000000000000006561511425433100232720ustar00rootroot00000000000000#!/bin/bash # added 2018-06-26 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "string syntax" reset_rules add_rule 'version=2' add_rule 'rule=:%f:string{"matching.permitted":[ {"class":"digit"} ], "matching.mode":"lazy"} %%r:rest%' execute '12:34 56' assert_output_json_eq '{ "r": ":34 56", "f": "12" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_string_lazy_matching.sh000077500000000000000000000016361511425433100227550ustar00rootroot00000000000000#!/bin/bash # added 2018-06-26 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "string syntax" reset_rules add_rule 'version=2' add_rule 'rule=:Rule-ID:%-:whitespace%% f:string{"matching.permitted":[ {"class":"digit"}, {"chars":"abcdefghijklmnopqrstuvwxyz"}, {"chars":"ABCDEFGHIJKLMNOPQRSTUVWXYZ"}, {"chars":"-"}, ], "quoting.escape.mode":"none", "matching.mode":"lazy"}%%resta:rest%' execute 'Rule-ID: XY7azl704-84a39894783423467a33f5b48bccd23c-a0n63i2\r\nQNas: ' assert_output_json_eq '{ "resta": "\\r\\nQNas: ", "f": "XY7azl704-84a39894783423467a33f5b48bccd23c-a0n63i2" }' execute 'Rule-ID: XY7azl704-84a39894783423467a33f5b48bccd23c-a0n63i2 LWL' assert_output_json_eq '{ "resta": " LWL", "f": "XY7azl704-84a39894783423467a33f5b48bccd23c-a0n63i2" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_string_perm_chars.sh000077500000000000000000000035401511425433100222430ustar00rootroot00000000000000#!/bin/bash # added 2015-09-02 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "string type with permitted chars" reset_rules add_rule 'version=2' add_rule 'rule=:a %f:string{"matching.permitted":"abc"}% b' execute 'a abc b' assert_output_json_eq '{"f": "abc"}' execute 'a abcd b' assert_output_json_eq '{"originalmsg": "a abcd b", "unparsed-data": "abcd b" }' execute 'a abbbbbcccbaaaa b' assert_output_json_eq '{"f": "abbbbbcccbaaaa"}' execute 'a "abc" b' assert_output_json_eq '{"f": "abc"}' echo "param array" reset_rules add_rule 'version=2' add_rule 'rule=:a %f:string{"matching.permitted":[ {"chars":"ab"}, {"chars":"c"} ]}% b' execute 'a abc b' assert_output_json_eq '{"f": "abc"}' reset_rules add_rule 'version=2' add_rule 'rule=:a %f:string{"matching.permitted":[ {"class":"digit"}, {"chars":"x"} ]}% b' execute 'a 12x3 b' assert_output_json_eq '{"f": "12x3"}' echo alpha reset_rules add_rule 'version=2' add_rule 'rule=:a %f:string{"matching.permitted":[ {"class":"alpha"} ]}% b' execute 'a abcdefghijklmnopqrstuvwxyZ b' assert_output_json_eq '{"f": "abcdefghijklmnopqrstuvwxyZ"}' execute 'a abcd1 b' assert_output_json_eq '{"originalmsg": "a abcd1 b", "unparsed-data": "abcd1 b" }' echo alnum reset_rules add_rule 'version=2' add_rule 'rule=:a %f:string{"matching.permitted":[ {"class":"alnum"} ]}% b' execute 'a abcdefghijklmnopqrstuvwxyZ b' assert_output_json_eq '{"f": "abcdefghijklmnopqrstuvwxyZ"}' execute 'a abcd1 b' assert_output_json_eq '{"f": "abcd1" }' execute 'a abcd1_ b' assert_output_json_eq '{ "originalmsg": "a abcd1_ b", "unparsed-data": "abcd1_ b" } ' cleanup_tmp_files liblognorm-2.0.8/tests/field_suffixed.sh000077500000000000000000000054231511425433100203510ustar00rootroot00000000000000#!/bin/bash # added 2015-02-25 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "field with one of many possible suffixes" add_rule 'rule=:gc reclaimed %eden_free:suffixed:,:b,kb,mb,gb:number% eden [surviver: %surviver_used:suffixed:;:kb;mb;gb;b:number%/%surviver_size:suffixed:|:b|kb|mb|gb:float%]' execute 'gc reclaimed 559mb eden [surviver: 95b/30.2mb]' assert_output_json_eq '{"eden_free": {"value": "559", "suffix":"mb"}, "surviver_used": {"value": "95", "suffix": "b"}, "surviver_size": {"value": "30.2", "suffix": "mb"}}' reset_rules add_rule 'rule=:gc reclaimed %eden_free:named_suffixed:size:unit:,:b,kb,mb,gb:number% eden [surviver: %surviver_used:named_suffixed:sz:u:;:kb;mb;gb;b:number%/%surviver_size:suffixed:|:b|kb|mb|gb:float%]' execute 'gc reclaimed 559mb eden [surviver: 95b/30.2mb]' assert_output_json_eq '{"eden_free": {"size": "559", "unit":"mb"}, "surviver_used": {"sz": "95", "u": "b"}, "surviver_size": {"value": "30.2", "suffix": "mb"}}' reset_rules add_rule 'rule=:gc reclaimed %eden_free:named_suffixed:size:unit:,:b,kb,mb,gb:interpret:int:number% from eden' execute 'gc reclaimed 559mb from eden' assert_output_json_eq '{"eden_free": {"size": 559, "unit":"mb"}}' reset_rules add_rule 'rule=:disk free: %free:named_suffixed:size:unit:,:\x25,gb:interpret:int:number%' execute 'disk free: 12%' assert_output_json_eq '{"free": {"size": 12, "unit":"%"}}' execute 'disk free: 130gb' assert_output_json_eq '{"free": {"size": 130, "unit":"gb"}}' reset_rules add_rule 'rule=:disk free: %free:named_suffixed:size:unit:\x3a:gb\x3a\x25:interpret:int:number%' execute 'disk free: 12%' assert_output_json_eq '{"free": {"size": 12, "unit":"%"}}' execute 'disk free: 130gb' assert_output_json_eq '{"free": {"size": 130, "unit":"gb"}}' reset_rules add_rule 'rule=:eden,surviver,old-gen available-capacity: %available_memory:tokenized:,:named_suffixed:size:unit:,:mb,gb:interpret:int:number%' execute 'eden,surviver,old-gen available-capacity: 400mb,40mb,1gb' assert_output_json_eq '{"available_memory": [{"size": 400, "unit":"mb"}, {"size": 40, "unit":"mb"}, {"size": 1, "unit":"gb"}]}' reset_rules add_rule 'rule=:eden,surviver,old-gen available-capacity: %available_memory:named_suffixed:size:unit:,:mb,gb:tokenized:,:interpret:int:number%' execute 'eden,surviver,old-gen available-capacity: 400,40,1024mb' assert_output_json_eq '{"available_memory": {"size": [400, 40, 1024], "unit":"mb"}}' reset_rules add_rule 'rule=:eden:surviver:old-gen available-capacity: %available_memory:named_suffixed:size:unit:,:mb,gb:tokenized:\x3a:interpret:int:number%' execute 'eden:surviver:old-gen available-capacity: 400:40:1024mb' assert_output_json_eq '{"available_memory": {"size": [400, 40, 1024], "unit":"mb"}}' cleanup_tmp_files liblognorm-2.0.8/tests/field_suffixed_with_invalid_ruledef.sh000077500000000000000000000043131511425433100246150ustar00rootroot00000000000000#!/bin/bash # added 2015-02-26 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "field with one of many possible suffixes, but invalid ruledef" add_rule 'rule=:reclaimed %eden_free:suffixe:,:b,kb,mb,gb:number% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' reset_rules add_rule 'rule=:reclaimed %eden_free:suffixed% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' reset_rules add_rule 'rule=:reclaimed %eden_free:suffixed:% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' reset_rules add_rule 'rule=:reclaimed %eden_free:suffixed:kb,mb% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' reset_rules add_rule 'rule=:reclaimed %eden_free:suffixed:kb,mb% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' reset_rules add_rule 'rule=:reclaimed %eden_free:suffixed:,:% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' reset_rules add_rule 'rule=:reclaimed %eden_free:suffixed:,:kb,mb% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' reset_rules add_rule 'rule=:reclaimed %eden_free:suffixed:,:kb,mb:% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' reset_rules add_rule 'rule=:reclaimed %eden_free:suffixed:,:kb,mb:floa% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' reset_rules add_rule 'rule=:reclaimed %eden_free:suffixed:,:kb,m:b:floa% eden' execute 'reclaimed 559mb eden' assert_output_json_eq '{ "originalmsg": "reclaimed 559mb eden", "unparsed-data": "559mb eden" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_tokenized.sh000077500000000000000000000023401511425433100205230ustar00rootroot00000000000000#!/bin/bash # added 2014-11-17 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "tokenized field" add_rule 'rule=:%arr:tokenized: , :word% %more:rest%' execute '123 , abc , 456 , def ijk789' assert_output_contains '"arr": [ "123", "abc", "456", "def" ]' assert_output_contains '"more": "ijk789"' reset_rules add_rule 'rule=:%ips:tokenized:, :ipv4% %text:rest%' execute '10.20.30.40, 50.60.70.80, 90.100.110.120 are blocked' assert_output_contains '"text": "are blocked"' assert_output_contains '"ips": [ "10.20.30.40", "50.60.70.80", "90.100.110.120" ]' reset_rules add_rule 'rule=:comma separated list of colon separated list of # separated numbers: %some_nos:tokenized:, :tokenized: \x3a :tokenized:#:number%' execute 'comma separated list of colon separated list of # separated numbers: 10, 20 : 30#40#50 : 60#70#80, 90 : 100' assert_output_contains '"some_nos": [ [ [ "10" ] ], [ [ "20" ], [ "30", "40", "50" ], [ "60", "70", "80" ] ], [ [ "90" ], [ "100" ] ] ]' reset_rules add_rule 'rule=:%arr:tokenized:\x3a:number% %more:rest%' execute '123:456:789 ijk789' assert_output_json_eq '{"arr": [ "123", "456", "789" ], "more": "ijk789"}' cleanup_tmp_files liblognorm-2.0.8/tests/field_tokenized_recursive.sh000077500000000000000000000051431511425433100226160ustar00rootroot00000000000000#!/bin/bash # added 2014-12-08 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "tokenized field with recursive field matching tokens" #recursive field inside tokenized field with default tail field add_rule 'rule=:%subnet_addr:ipv4%/%subnet_mask:number%%tail:rest%' add_rule 'rule=:%ip_addr:ipv4%%tail:rest%' add_rule 'rule=:blocked inbound via: %via_ip:ipv4% from: %addresses:tokenized:, :recursive% to %server_ip:ipv4%' execute 'blocked inbound via: 192.168.1.1 from: 1.2.3.4, 5.6.16.0/12, 8.9.10.11, 12.13.14.15, 16.17.18.0/8, 19.20.21.24/3 to 192.168.1.5' assert_output_json_eq '{ "addresses": [ {"ip_addr": "1.2.3.4"}, {"subnet_addr": "5.6.16.0", "subnet_mask": "12"}, {"ip_addr": "8.9.10.11"}, {"ip_addr": "12.13.14.15"}, {"subnet_addr": "16.17.18.0", "subnet_mask": "8"}, {"subnet_addr": "19.20.21.24", "subnet_mask": "3"}], "server_ip": "192.168.1.5", "via_ip": "192.168.1.1"}' reset_rules #recursive field inside tokenized field with default tail field reset_rules add_rule 'rule=:%subnet_addr:ipv4%/%subnet_mask:number%%remains:rest%' add_rule 'rule=:%ip_addr:ipv4%%remains:rest%' add_rule 'rule=:blocked inbound via: %via_ip:ipv4% from: %addresses:tokenized:, :recursive:remains% to %server_ip:ipv4%' execute 'blocked inbound via: 192.168.1.1 from: 1.2.3.4, 5.6.16.0/12, 8.9.10.11, 12.13.14.15, 16.17.18.0/8, 19.20.21.24/3 to 192.168.1.5' assert_output_json_eq '{ "addresses": [ {"ip_addr": "1.2.3.4"}, {"subnet_addr": "5.6.16.0", "subnet_mask": "12"}, {"ip_addr": "8.9.10.11"}, {"ip_addr": "12.13.14.15"}, {"subnet_addr": "16.17.18.0", "subnet_mask": "8"}, {"subnet_addr": "19.20.21.24", "subnet_mask": "3"}], "server_ip": "192.168.1.5", "via_ip": "192.168.1.1"}' #recursive field inside tokenized field with default tail field reset_rules 'net' add_rule 'rule=:%subnet_addr:ipv4%/%subnet_mask:number%%remains:rest%' 'net' add_rule 'rule=:%ip_addr:ipv4%%remains:rest%' 'net' reset_rules add_rule 'rule=:blocked inbound via: %via_ip:ipv4% from: %addresses:tokenized:, :descent:./net.rulebase:remains% to %server_ip:ipv4%' execute 'blocked inbound via: 192.168.1.1 from: 1.2.3.4, 5.6.16.0/12, 8.9.10.11, 12.13.14.15, 16.17.18.0/8, 19.20.21.24/3 to 192.168.1.5' assert_output_json_eq '{ "addresses": [ {"ip_addr": "1.2.3.4"}, {"subnet_addr": "5.6.16.0", "subnet_mask": "12"}, {"ip_addr": "8.9.10.11"}, {"ip_addr": "12.13.14.15"}, {"subnet_addr": "16.17.18.0", "subnet_mask": "8"}, {"subnet_addr": "19.20.21.24", "subnet_mask": "3"}], "server_ip": "192.168.1.5", "via_ip": "192.168.1.1"}' cleanup_tmp_files liblognorm-2.0.8/tests/field_tokenized_with_invalid_ruledef.sh000077500000000000000000000026121511425433100247740ustar00rootroot00000000000000#!/bin/bash # added 2014-11-18 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "tokenized field with invalid rule definition" add_rule 'rule=:%arr:tokenized%' execute '123 abc 456 def' assert_output_contains '"unparsed-data": "123 abc 456 def"' assert_output_contains '"originalmsg": "123 abc 456 def"' reset_rules add_rule 'rule=:%arr:tokenized: %' execute '123 abc 456 def' assert_output_contains '"unparsed-data": "123 abc 456 def"' assert_output_contains '"originalmsg": "123 abc 456 def"' reset_rules add_rule 'rule=:%arr:tokenized:quux:%' execute '123 abc 456 def' assert_output_contains '"unparsed-data": "123 abc 456 def"' assert_output_contains '"originalmsg": "123 abc 456 def"' reset_rules add_rule 'rule=:%arr:tokenized:quux:some_non_existent_type%' execute '123 abc 456 def' assert_output_contains '"unparsed-data": "123 abc 456 def"' assert_output_contains '"originalmsg": "123 abc 456 def"' reset_rules add_rule 'rule=:%arr:tokenized:quux:some_non_existent_type:%' execute '123 abc 456 def' assert_output_contains '"unparsed-data": "123 abc 456 def"' assert_output_contains '"originalmsg": "123 abc 456 def"' reset_rules add_rule 'rule=:%arr:tokenized::::%%%%' execute '123 abc 456 def' assert_output_contains '"unparsed-data": "123 abc 456 def"' assert_output_contains '"originalmsg": "123 abc 456 def"' cleanup_tmp_files liblognorm-2.0.8/tests/field_tokenized_with_regex.sh000077500000000000000000000015701511425433100227540ustar00rootroot00000000000000#!/bin/bash # added 2014-11-17 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 #test that tokenized disabled regex if parent context has it disabled . $srcdir/exec.sh no_solaris10 test_def $0 "tokenized field with regex based field" add_rule 'rule=:%parts:tokenized:,:regex:[^, ]+% %text:rest%' execute '123,abc,456,def foo bar' assert_output_contains '"unparsed-data": "123,abc,456,def foo bar"' assert_output_contains '"originalmsg": "123,abc,456,def foo bar"' #and then enables it when parent context has it enabled export ln_opts='-oallowRegex' . $srcdir/exec.sh test_def $0 "tokenized field with regex based field" add_rule 'rule=:%parts:tokenized:,:regex:[^, ]+% %text:rest%' execute '123,abc,456,def foo bar' assert_output_contains '"parts": [ "123", "abc", "456", "def" ]' assert_output_contains '"text": "foo bar"' cleanup_tmp_files liblognorm-2.0.8/tests/field_v2-iptables.sh000077500000000000000000000047721511425433100206720ustar00rootroot00000000000000#!/bin/bash # added 2015-04-30 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "v2-iptables field" add_rule 'version=2' add_rule 'rule=:iptables output denied: %field:v2-iptables%' # first, a real-world case execute 'iptables output denied: IN= OUT=eth0 SRC=176.9.56.141 DST=168.192.14.3 LEN=32 TOS=0x00 PREC=0x00 TTL=64 ID=39110 DF PROTO=UDP SPT=49564 DPT=2010 LEN=12' assert_output_json_eq '{ "field": { "IN": "", "OUT": "eth0", "SRC": "176.9.56.141", "DST": "168.192.14.3", "LEN": "12", "TOS": "0x00", "PREC": "0x00", "TTL": "64", "ID": "39110", "DF": null, "PROTO": "UDP", "SPT": "49564", "DPT": "2010" } }' # now some more "fabricated" cases for better readable test reset_rules add_rule 'rule=:iptables: %field:v2-iptables%' execute 'iptables: IN=value SECOND=test' assert_output_json_eq '{ "field": { "IN": "value", "SECOND": "test" }} }' execute 'iptables: IN= SECOND=test' assert_output_json_eq '{ "field": { "IN": ""} }' execute 'iptables: IN SECOND=test' assert_output_json_eq '{ "field": { "IN": null} }' execute 'iptables: IN=invalue OUT=outvalue' assert_output_json_eq '{ "field": { "IN": "invalue", "OUT": "outvalue" } }' execute 'iptables: IN= OUT=outvalue' assert_output_json_eq '{ "field": { "IN": "", "OUT": "outvalue" } }' execute 'iptables: IN OUT=outvalue' assert_output_json_eq '{ "field": { "IN": null, "OUT": "outvalue" } }' # #check cases where parsing failure must occur # echo verify failure cases # lower case is not permitted execute 'iptables: in=value' assert_output_json_eq '{ "originalmsg": "iptables: in=value", "unparsed-data": "in=value" }' execute 'iptables: in=' assert_output_json_eq '{ "originalmsg": "iptables: in=", "unparsed-data": "in=" }' execute 'iptables: in' assert_output_json_eq '{ "originalmsg": "iptables: in", "unparsed-data": "in" }' execute 'iptables: IN' # single field is NOT permitted! assert_output_json_eq '{ "originalmsg": "iptables: IN", "unparsed-data": "IN" }' # multiple spaces between n=v pairs are not permitted execute 'iptables: IN=invalue OUT=outvalue' assert_output_json_eq '{ "originalmsg": "iptables: IN=invalue OUT=outvalue", "unparsed-data": "IN=invalue OUT=outvalue" }' execute 'iptables: IN= OUT=outvalue' assert_output_json_eq '{ "originalmsg": "iptables: IN= OUT=outvalue", "unparsed-data": "IN= OUT=outvalue" }' execute 'iptables: IN OUT=outvalue' assert_output_json_eq '{ "originalmsg": "iptables: IN OUT=outvalue", "unparsed-data": "IN OUT=outvalue" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_v2-iptables_jsoncnf.sh000077500000000000000000000050441511425433100224030ustar00rootroot00000000000000#!/bin/bash # added 2015-04-30 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "v2-iptables field" add_rule 'version=2' add_rule 'rule=:iptables output denied: %{"name":"field", "type":"v2-iptables"}%' # first, a real-world case execute 'iptables output denied: IN= OUT=eth0 SRC=176.9.56.141 DST=168.192.14.3 LEN=32 TOS=0x00 PREC=0x00 TTL=64 ID=39110 DF PROTO=UDP SPT=49564 DPT=2010 LEN=12' assert_output_json_eq '{ "field": { "IN": "", "OUT": "eth0", "SRC": "176.9.56.141", "DST": "168.192.14.3", "LEN": "12", "TOS": "0x00", "PREC": "0x00", "TTL": "64", "ID": "39110", "DF": null, "PROTO": "UDP", "SPT": "49564", "DPT": "2010" } }' # now some more "fabricated" cases for better readable test reset_rules add_rule 'version=2' add_rule 'rule=:iptables: %field:v2-iptables%' execute 'iptables: IN=value SECOND=test' assert_output_json_eq '{ "field": { "IN": "value", "SECOND": "test" }} }' execute 'iptables: IN= SECOND=test' assert_output_json_eq '{ "field": { "IN": ""} }' execute 'iptables: IN SECOND=test' assert_output_json_eq '{ "field": { "IN": null} }' execute 'iptables: IN=invalue OUT=outvalue' assert_output_json_eq '{ "field": { "IN": "invalue", "OUT": "outvalue" } }' execute 'iptables: IN= OUT=outvalue' assert_output_json_eq '{ "field": { "IN": "", "OUT": "outvalue" } }' execute 'iptables: IN OUT=outvalue' assert_output_json_eq '{ "field": { "IN": null, "OUT": "outvalue" } }' # #check cases where parsing failure must occur # echo verify failure cases # lower case is not permitted execute 'iptables: in=value' assert_output_json_eq '{ "originalmsg": "iptables: in=value", "unparsed-data": "in=value" }' execute 'iptables: in=' assert_output_json_eq '{ "originalmsg": "iptables: in=", "unparsed-data": "in=" }' execute 'iptables: in' assert_output_json_eq '{ "originalmsg": "iptables: in", "unparsed-data": "in" }' execute 'iptables: IN' # single field is NOT permitted! assert_output_json_eq '{ "originalmsg": "iptables: IN", "unparsed-data": "IN" }' # multiple spaces between n=v pairs are not permitted execute 'iptables: IN=invalue OUT=outvalue' assert_output_json_eq '{ "originalmsg": "iptables: IN=invalue OUT=outvalue", "unparsed-data": "IN=invalue OUT=outvalue" }' execute 'iptables: IN= OUT=outvalue' assert_output_json_eq '{ "originalmsg": "iptables: IN= OUT=outvalue", "unparsed-data": "IN= OUT=outvalue" }' execute 'iptables: IN OUT=outvalue' assert_output_json_eq '{ "originalmsg": "iptables: IN OUT=outvalue", "unparsed-data": "IN OUT=outvalue" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_v2-iptables_v1.sh000077500000000000000000000047451511425433100213000ustar00rootroot00000000000000#!/bin/bash # added 2015-04-30 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "v2-iptables field" add_rule 'rule=:iptables output denied: %field:v2-iptables%' # first, a real-world case execute 'iptables output denied: IN= OUT=eth0 SRC=176.9.56.141 DST=168.192.14.3 LEN=32 TOS=0x00 PREC=0x00 TTL=64 ID=39110 DF PROTO=UDP SPT=49564 DPT=2010 LEN=12' assert_output_json_eq '{ "field": { "IN": "", "OUT": "eth0", "SRC": "176.9.56.141", "DST": "168.192.14.3", "LEN": "12", "TOS": "0x00", "PREC": "0x00", "TTL": "64", "ID": "39110", "DF": null, "PROTO": "UDP", "SPT": "49564", "DPT": "2010" } }' # now some more "fabricated" cases for better readable test reset_rules add_rule 'rule=:iptables: %field:v2-iptables%' execute 'iptables: IN=value SECOND=test' assert_output_json_eq '{ "field": { "IN": "value", "SECOND": "test" }} }' execute 'iptables: IN= SECOND=test' assert_output_json_eq '{ "field": { "IN": ""} }' execute 'iptables: IN SECOND=test' assert_output_json_eq '{ "field": { "IN": null} }' execute 'iptables: IN=invalue OUT=outvalue' assert_output_json_eq '{ "field": { "IN": "invalue", "OUT": "outvalue" } }' execute 'iptables: IN= OUT=outvalue' assert_output_json_eq '{ "field": { "IN": "", "OUT": "outvalue" } }' execute 'iptables: IN OUT=outvalue' assert_output_json_eq '{ "field": { "IN": null, "OUT": "outvalue" } }' # #check cases where parsing failure must occur # echo verify failure cases # lower case is not permitted execute 'iptables: in=value' assert_output_json_eq '{ "originalmsg": "iptables: in=value", "unparsed-data": "in=value" }' execute 'iptables: in=' assert_output_json_eq '{ "originalmsg": "iptables: in=", "unparsed-data": "in=" }' execute 'iptables: in' assert_output_json_eq '{ "originalmsg": "iptables: in", "unparsed-data": "in" }' execute 'iptables: IN' # single field is NOT permitted! assert_output_json_eq '{ "originalmsg": "iptables: IN", "unparsed-data": "IN" }' # multiple spaces between n=v pairs are not permitted execute 'iptables: IN=invalue OUT=outvalue' assert_output_json_eq '{ "originalmsg": "iptables: IN=invalue OUT=outvalue", "unparsed-data": "IN=invalue OUT=outvalue" }' execute 'iptables: IN= OUT=outvalue' assert_output_json_eq '{ "originalmsg": "iptables: IN= OUT=outvalue", "unparsed-data": "IN= OUT=outvalue" }' execute 'iptables: IN OUT=outvalue' assert_output_json_eq '{ "originalmsg": "iptables: IN OUT=outvalue", "unparsed-data": "IN OUT=outvalue" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_whitespace.sh000077500000000000000000000021551511425433100206670ustar00rootroot00000000000000#!/bin/bash # added 2015-03-12 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "whitespace parser" # the "word" parser unfortunately treats everything except # a SP as being in the word. So a HT inside a word is # permitted, which does not work well with what we # want to test here. to solve this problem, we use op-quoted-string. # However, we must actually quote the samples with HT, because # that parser also treats HT as being part of the word. But thanks # to the quotes, we can force it to not do that. # rgerhards, 2015-04-30 add_rule 'version=2' add_rule 'rule=:%a:op-quoted-string%%-:whitespace%%b:op-quoted-string%' execute 'word1 word2' # multiple spaces assert_output_json_eq '{ "b": "word2", "a": "word1" }' execute 'word1 word2' # single space assert_output_json_eq '{ "b": "word2", "a": "word1" }' execute '"word1" "word2"' # tab (US-ASCII HT) assert_output_json_eq '{ "b": "word2", "a": "word1" }' execute '"word1" "word2"' # mix of tab and spaces assert_output_json_eq '{ "b": "word2", "a": "word1" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_whitespace_jsoncnf.sh000077500000000000000000000022541511425433100224070ustar00rootroot00000000000000#!/bin/bash # added 2015-03-12 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "whitespace parser" # the "word" parser unfortunately treats everything except # a SP as being in the word. So a HT inside a word is # permitted, which does not work well with what we # want to test here. to solve this problem, we use op-quoted-string. # However, we must actually quote the samples with HT, because # that parser also treats HT as being part of the word. But thanks # to the quotes, we can force it to not do that. # rgerhards, 2015-04-30 add_rule 'version=2' add_rule 'rule=:%{"name":"a", "type":"op-quoted-string"}%%{"name":"-", "type":"whitespace"}%%{"name":"b", "type":"op-quoted-string"}%' execute 'word1 word2' # multiple spaces assert_output_json_eq '{ "b": "word2", "a": "word1" }' execute 'word1 word2' # single space assert_output_json_eq '{ "b": "word2", "a": "word1" }' execute '"word1" "word2"' # tab (US-ASCII HT) assert_output_json_eq '{ "b": "word2", "a": "word1" }' execute '"word1" "word2"' # mix of tab and spaces assert_output_json_eq '{ "b": "word2", "a": "word1" }' cleanup_tmp_files liblognorm-2.0.8/tests/field_whitespace_v1.sh000077500000000000000000000021301511425433100212660ustar00rootroot00000000000000#!/bin/bash # added 2015-03-12 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "whitespace parser" # the "word" parser unfortunately treats everything except # a SP as being in the word. So a HT inside a word is # permitted, which does not work well with what we # want to test here. to solve this problem, we use op-quoted-string. # However, we must actually quote the samples with HT, because # that parser also treats HT as being part of the word. But thanks # to the quotes, we can force it to not do that. # rgerhards, 2015-04-30 add_rule 'rule=:%a:op-quoted-string%%-:whitespace%%b:op-quoted-string%' execute 'word1 word2' # multiple spaces assert_output_json_eq '{ "b": "word2", "a": "word1" }' execute 'word1 word2' # single space assert_output_json_eq '{ "b": "word2", "a": "word1" }' execute '"word1" "word2"' # tab (US-ASCII HT) assert_output_json_eq '{ "b": "word2", "a": "word1" }' execute '"word1" "word2"' # mix of tab and spaces assert_output_json_eq '{ "b": "word2", "a": "word1" }' cleanup_tmp_files liblognorm-2.0.8/tests/include.sh000077500000000000000000000007661511425433100170210ustar00rootroot00000000000000#!/bin/bash # added 2015-08-28 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "include (success case)" reset_rules add_rule 'version=2' add_rule 'include=inc.rulebase' reset_rules inc add_rule 'version=2' inc add_rule 'rule=:%field:mac48%' inc execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"field": "f0:f6:1c:5f:cc:a2"}' # single test is sufficient, because that only works if the include # worked ;) cleanup_tmp_files liblognorm-2.0.8/tests/include_RULEBASES.sh000077500000000000000000000011751511425433100204210ustar00rootroot00000000000000#!/bin/bash # added 2015-08-28 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "include (via LIBLOGNORM_RULEBASES directory)" reset_rules add_rule 'version=2' add_rule 'include=inc.rulebase' reset_rules inc add_rule 'version=2' inc add_rule 'rule=:%field:mac48%' inc mv inc.rulebase /tmp export LIBLOGNORM_RULEBASES=/tmp execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"field": "f0:f6:1c:5f:cc:a2"}' export LIBLOGNORM_RULEBASES=/tmp/ execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"field": "f0:f6:1c:5f:cc:a2"}' rm /tmp/inc.rulebase cleanup_tmp_files liblognorm-2.0.8/tests/json_eq.c000066400000000000000000000053431511425433100166350ustar00rootroot00000000000000#include "config.h" #include #include #include #include #include #include "internal.h" typedef struct json_object obj; static int eq(obj* expected, obj* actual); static int obj_eq(obj* expected, obj* actual) { int eql = 1; struct json_object_iterator it = json_object_iter_begin(expected); struct json_object_iterator itEnd = json_object_iter_end(expected); while (!json_object_iter_equal(&it, &itEnd)) { obj *actual_val; json_object_object_get_ex(actual, json_object_iter_peek_name(&it), &actual_val); eql &= eq(json_object_iter_peek_value(&it), actual_val); json_object_iter_next(&it); } return eql; } static int arr_eq(obj* expected, obj* actual) { int eql = 1; int expected_len = json_object_array_length(expected); int actual_len = json_object_array_length(actual); if (expected_len != actual_len) return 0; for (int i = 0; i < expected_len; i++) { obj* exp = json_object_array_get_idx(expected, i); obj* act = json_object_array_get_idx(actual, i); eql &= eq(exp, act); } return eql; } static int str_eq(obj* expected, obj* actual) { const char* exp_str = json_object_to_json_string(expected); const char* act_str = json_object_to_json_string(actual); return strcmp(exp_str, act_str) == 0; } static int eq(obj* expected, obj* actual) { if (expected == NULL && actual == NULL) { return 1; } else if (expected == NULL) { return 0; } else if (actual == NULL) { return 0; } enum json_type expected_type = json_object_get_type(expected); enum json_type actual_type = json_object_get_type(actual); if (expected_type != actual_type) return 0; switch(expected_type) { case json_type_null: return 1; case json_type_boolean: return json_object_get_boolean(expected) == json_object_get_boolean(actual); case json_type_double: return (fabs(json_object_get_double(expected) - json_object_get_double(actual)) < 0.001); case json_type_int: return json_object_get_int64(expected) == json_object_get_int64(actual); case json_type_object: return obj_eq(expected, actual); case json_type_array: return arr_eq(expected, actual); case json_type_string: return str_eq(expected, actual); default: fprintf(stderr, "unexpected type in %s:%d\n", __FILE__, __LINE__); abort(); } return 0; } int main(int argc, char** argv) { if (argc != 3) { fprintf(stderr, "expected and actual json not given, number of args was: %d\n", argc); exit(100); } obj* expected = json_tokener_parse(argv[1]); obj* actual = json_tokener_parse(argv[2]); int result = eq(expected, actual) ? 0 : 1; json_object_put(expected); json_object_put(actual); if (result != 0) { printf("JSONs weren't equal. \n\tExpected: \n\t\t%s\n\tActual: \n\t\t%s\n", argv[1], argv[2]); } return result; } liblognorm-2.0.8/tests/literal.sh000077500000000000000000000016461511425433100170300ustar00rootroot00000000000000#!/bin/bash # added 2016-12-21 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "using names with literal" add_rule 'version=2' add_rule 'rule=:%{"type":"literal", "text":"a", "name":"var"}%' execute 'a' assert_output_json_eq '{ "var": "a" }' reset_rules add_rule 'version=2' add_rule 'rule=:Test %{"type":"literal", "text":"a", "name":"var"}%' execute 'Test a' assert_output_json_eq '{ "var": "a" }' reset_rules add_rule 'version=2' add_rule 'rule=:Test %{"type":"literal", "text":"a", "name":"var"}% End' execute 'Test a End' assert_output_json_eq '{ "var": "a" }' reset_rules add_rule 'version=2' add_rule 'rule=:a %[{"name":"num", "type":"number"}, {"name":"colon", "type":"literal", "text":":"}, {"name":"hex", "type":"hexnumber"}]% b' execute 'a 4711:0x4712 b' assert_output_json_eq '{ "hex": "0x4712", "colon": ":", "num": "4711" }' cleanup_tmp_files liblognorm-2.0.8/tests/lognormalizer-invld-call.sh000077500000000000000000000005201511425433100222710ustar00rootroot00000000000000#!/bin/bash # This file is part of the liblognorm project, released under ASL 2.0 echo running test $0 if ../src/lognormalizer ; then echo "FAIL: loganalyzer did not detect missing rulebase" exit 1 fi if ../src/lognormalizer -r test -R test ; then echo "FAIL: loganalyzer did not detect both -r and -R given" exit 1 fi liblognorm-2.0.8/tests/missing_line_ending.sh000077500000000000000000000013251511425433100213720ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "missing line ending" reset_rules add_rule 'version=2' add_rule_no_LF 'rule=:%field:mac48%' execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"field": "f0:f6:1c:5f:cc:a2"}' execute 'f0-f6-1c-5f-cc-a2' assert_output_json_eq '{"field": "f0-f6-1c-5f-cc-a2"}' # things that need to NOT match execute 'f0-f6:1c:5f:cc-a2' assert_output_json_eq '{ "originalmsg": "f0-f6:1c:5f:cc-a2", "unparsed-data": "f0-f6:1c:5f:cc-a2" }' execute 'f0:f6:1c:xf:cc:a2' assert_output_json_eq '{ "originalmsg": "f0:f6:1c:xf:cc:a2", "unparsed-data": "f0:f6:1c:xf:cc:a2" }' cleanup_tmp_files liblognorm-2.0.8/tests/missing_line_ending_v1.sh000077500000000000000000000013041511425433100217750ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "missing line ending (v1)" reset_rules add_rule_no_LF 'rule=:%field:mac48%' execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"field": "f0:f6:1c:5f:cc:a2"}' execute 'f0-f6-1c-5f-cc-a2' assert_output_json_eq '{"field": "f0-f6-1c-5f-cc-a2"}' # things that need to NOT match execute 'f0-f6:1c:5f:cc-a2' assert_output_json_eq '{ "originalmsg": "f0-f6:1c:5f:cc-a2", "unparsed-data": "f0-f6:1c:5f:cc-a2" }' execute 'f0:f6:1c:xf:cc:a2' assert_output_json_eq '{ "originalmsg": "f0:f6:1c:xf:cc:a2", "unparsed-data": "f0:f6:1c:xf:cc:a2" }' cleanup_tmp_files liblognorm-2.0.8/tests/names.sh000077500000000000000000000022371511425433100164740ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "using names with literal" add_rule 'version=2' add_rule 'rule=:a %[{"name":"num", "type":"number"}, {"type":"literal", "text":":"}, {"name":"hex", "type":"hexnumber"}]% b' execute 'a 4711:0x4712 b' assert_output_json_eq '{ "hex": "0x4712", "num": "4711" }' reset_rules add_rule 'version=2' add_rule 'rule=:a %[{"name":"num", "type":"number"}, {"name":"literal", "type":"literal", "text":":"}, {"name":"hex", "type":"hexnumber"}]% b' execute 'a 4711:0x4712 b' assert_output_json_eq '{ "hex": "0x4712", "num": "4711" }' # check that "-" is still discarded reset_rules add_rule 'version=2' add_rule 'rule=:a %[{"name":"num", "type":"number"}, {"name":"-", "type":"literal", "text":":"}, {"name":"hex", "type":"hexnumber"}]% b' execute 'a 4711:0x4712 b' assert_output_json_eq '{ "hex": "0x4712", "num": "4711" }' # now let's check old style. Here we need "-". reset_rules add_rule 'version=2' add_rule 'rule=:a %-:number%:%hex:hexnumber% b' execute 'a 4711:0x4712 b' assert_output_json_eq '{ "hex": "0x4712" }' cleanup_tmp_files liblognorm-2.0.8/tests/options.sh.in000066400000000000000000000003311511425433100174570ustar00rootroot00000000000000use_valgrind=@VALGRIND@ echo "Using valgrind: $use_valgrind" if [ $use_valgrind == "yes" ]; then cmd="valgrind --error-exitcode=191 --malloc-fill=ff --free-fill=fe --leak-check=full --trace-children=yes $cmd" fi liblognorm-2.0.8/tests/parser_LF.sh000077500000000000000000000012451511425433100172440ustar00rootroot00000000000000#!/bin/bash # added 2015-07-15 by Rainer Gerhards # This checks if whitespace inside parser definitions is properly treated # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "LF in parser definition" add_rule 'rule=:here is a number % num:hexnumber % in hex form' execute 'here is a number 0x1234 in hex form' assert_output_json_eq '{"num": "0x1234"}' #check cases where parsing failure must occur execute 'here is a number 0x1234in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0x1234in hex form", "unparsed-data": "0x1234in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/parser_LF_jsoncnf.sh000077500000000000000000000013031511425433100207570ustar00rootroot00000000000000#!/bin/bash # added 2015-07-15 by Rainer Gerhards # This checks if whitespace inside parser definitions is properly treated # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "LF in parser definition" add_rule 'version=2' add_rule 'rule=:here is a number %{ "name":"num", "type":"hexnumber" }% in hex form' execute 'here is a number 0x1234 in hex form' assert_output_json_eq '{"num": "0x1234"}' #check cases where parsing failure must occur execute 'here is a number 0x1234in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0x1234in hex form", "unparsed-data": "0x1234in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/parser_prios.sh000077500000000000000000000021331511425433100200740ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "parser priorities, simple case" add_rule 'version=2' add_rule 'rule=:%{"name":"field", "type":"mac48"}%' add_rule 'rule=:%{"name":"rest", "type":"rest"}%' execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"field": "f0:f6:1c:5f:cc:a2"}' execute 'f0-f6-1c-5f-cc-a2' assert_output_json_eq '{"field": "f0-f6-1c-5f-cc-a2"}' # things that need to match rest execute 'f0-f6:1c:5f:cc-a2' assert_output_json_eq '{ "rest": "f0-f6:1c:5f:cc-a2" }' # now the same with inverted priorities. We should now always have # rest matches. reset_rules add_rule 'version=2' add_rule 'rule=:%{"name":"field", "type":"mac48", "priority":100}%' add_rule 'rule=:%{"name":"rest", "type":"rest", "priority":10}%' execute 'f0:f6:1c:5f:cc:a2' assert_output_json_eq '{"rest": "f0:f6:1c:5f:cc:a2"}' execute 'f0-f6-1c-5f-cc-a2' assert_output_json_eq '{"rest": "f0-f6-1c-5f-cc-a2"}' execute 'f0-f6:1c:5f:cc-a2' assert_output_json_eq '{ "rest": "f0-f6:1c:5f:cc-a2" }' cleanup_tmp_files liblognorm-2.0.8/tests/parser_whitespace.sh000077500000000000000000000012211511425433100210710ustar00rootroot00000000000000#!/bin/bash # added 2015-07-15 by Rainer Gerhards # This checks if whitespace inside parser definitions is properly treated # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh no_solaris10 test_def $0 "whitespace in parser definition" add_rule 'rule=:here is a number % num:hexnumber % in hex form' execute 'here is a number 0x1234 in hex form' assert_output_json_eq '{"num": "0x1234"}' #check cases where parsing failure must occur execute 'here is a number 0x1234in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0x1234in hex form", "unparsed-data": "0x1234in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/parser_whitespace_jsoncnf.sh000077500000000000000000000012571511425433100226220ustar00rootroot00000000000000#!/bin/bash # added 2015-07-15 by Rainer Gerhards # This checks if whitespace inside parser definitions is properly treated # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "whitespace in parser definition" add_rule 'version=2' add_rule 'rule=:here is a number % {"name":"num", "type":"hexnumber"} % in hex form' execute 'here is a number 0x1234 in hex form' assert_output_json_eq '{"num": "0x1234"}' #check cases where parsing failure must occur execute 'here is a number 0x1234in hex form' assert_output_json_eq '{ "originalmsg": "here is a number 0x1234in hex form", "unparsed-data": "0x1234in hex form" }' cleanup_tmp_files liblognorm-2.0.8/tests/repeat_alternative_nested.sh000077500000000000000000000024021511425433100226030ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple alternative syntax" add_rule 'version=2' add_rule 'rule=:a %{"name":"numbers", "type":"repeat", "parser": { "type":"alternative", "parser": [ [ {"type":"number", "name":"n1"}, {"type":"literal", "text":":"}, {"type":"number", "name":"n2"}, ], {"type":"hexnumber", "name":"hex"} ] }, "while":[ {"type":"literal", "text":", "} ] }% b' execute 'a 1:2, 3:4, 5:6, 7:8 b' assert_output_json_eq '{ "numbers": [ { "n2": "2", "n1": "1" }, { "n2": "4", "n1": "3" }, { "n2": "6", "n1": "5" }, { "n2": "8", "n1": "7" } ] }' execute 'a 0x4711 b' assert_output_json_eq '{ "numbers": [ { "hex": "0x4711" } ] }' # note: 0x4711, 1:2 does not work because hexnumber expects a SP after # the number! Thus we use the reverse. We could add this case once # we have added an option for more relaxed matching to hexnumber. execute 'a 1:2, 0x4711 b' assert_output_json_eq '{ "numbers": [ { "n2": "2", "n1": "1" }, { "hex": "0x4711" } ] }' cleanup_tmp_files liblognorm-2.0.8/tests/repeat_mismatch_in_while.sh000077500000000000000000000051331511425433100224120ustar00rootroot00000000000000#!/bin/bash # added 2015-08-26 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 # This is based on a practical support case, see # https://github.com/rsyslog/liblognorm/issues/130 . $srcdir/exec.sh test_def $0 "repeat with mismatch in parser part" reset_rules add_rule 'version=2' add_rule 'prefix=%timestamp:date-rfc3164% %hostname:word%' add_rule 'rule=cisco,fwblock: \x25ASA-6-106015\x3a Deny %proto:word% (no connection) from %source:cisco-interface-spec% to %dest:cisco-interface-spec% flags %flags:repeat{ "parser": {"type":"word", "name":"."}, "while":{"type":"literal", "text":" "} }% on interface %srciface:word%' echo step 1 execute 'Aug 18 13:18:45 192.168.99.2 %ASA-6-106015: Deny TCP (no connection) from 173.252.88.66/443 to 76.79.249.222/52746 flags RST on interface outside' assert_output_json_eq '{ "originalmsg": "Aug 18 13:18:45 192.168.99.2 %ASA-6-106015: Deny TCP (no connection) from 173.252.88.66\/443 to 76.79.249.222\/52746 flags RST on interface outside", "unparsed-data": "RST on interface outside" }' # now check case where we permit a mismatch inside the parser part and still # accept this as valid. This is needed for some use cases. See github # issue mentioned above for more details. # Note: there is something odd with the testbench driver: I cannot use two # consecutive spaces reset_rules add_rule 'version=2' add_rule 'prefix=%timestamp:date-rfc3164% %hostname:word%' add_rule 'rule=cisco,fwblock: \x25ASA-6-106015\x3a Deny %proto:word% (no connection) from %source:cisco-interface-spec% to %dest:cisco-interface-spec% flags %flags:repeat{ "option.permitMismatchInParser":true, "parser": {"type":"word", "name":"."}, "while":{"type":"literal", "text":" "} }%\x20 on interface %srciface:word%' echo step 2 execute 'Aug 18 13:18:45 192.168.99.2 %ASA-6-106015: Deny TCP (no connection) from 173.252.88.66/443 to 76.79.249.222/52746 flags RST on interface outside' assert_output_json_eq '{ "srciface": "outside", "flags": [ "RST" ], "dest": { "ip": "76.79.249.222", "port": "52746" }, "source": { "ip": "173.252.88.66", "port": "443" }, "proto": "TCP", "hostname": "192.168.99.2", "timestamp": "Aug 18 13:18:45" }' echo step 3 execute 'Aug 18 13:18:45 192.168.99.2 %ASA-6-106015: Deny TCP (no connection) from 173.252.88.66/443 to 76.79.249.222/52746 flags RST XST on interface outside' assert_output_json_eq '{ "srciface": "outside", "flags": [ "RST", "XST" ], "dest": { "ip": "76.79.249.222", "port": "52746" }, "source": { "ip": "173.252.88.66", "port": "443" }, "proto": "TCP", "hostname": "192.168.99.2", "timestamp": "Aug 18 13:18:45" }' cleanup_tmp_files liblognorm-2.0.8/tests/repeat_simple.sh000077500000000000000000000012521511425433100202160ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple repeat syntax" add_rule 'version=2' add_rule 'rule=:a %{"name":"numbers", "type":"repeat", "parser":[ {"name":"n1", "type":"number"}, {"type":"literal", "text":":"}, {"name":"n2", "type":"number"} ], "while":[ {"type":"literal", "text":", "} ] }% b %w:word% ' execute 'a 1:2, 3:4, 5:6, 7:8 b test' assert_output_json_eq '{ "w": "test", "numbers": [ { "n2": "2", "n1": "1" }, { "n2": "4", "n1": "3" }, { "n2": "6", "n1": "5" }, { "n2": "8", "n1": "7" } ] }' cleanup_tmp_files liblognorm-2.0.8/tests/repeat_very_simple.sh000077500000000000000000000010361511425433100212630ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "very simple repeat syntax" add_rule 'version=2' add_rule 'rule=:a %{"name":"numbers", "type":"repeat", "parser": {"name":"n", "type":"number"}, "while": {"type":"literal", "text":", "} }% b %w:word% ' execute 'a 1, 2, 3, 4 b test' assert_output_json_eq '{ "w": "test", "numbers": [ { "n": "1" }, { "n": "2" }, { "n": "3" }, { "n": "4" } ] }' cleanup_tmp_files liblognorm-2.0.8/tests/repeat_while_alternative.sh000077500000000000000000000014531511425433100224360ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "repeat syntax with alternative terminators" add_rule 'version=2' add_rule 'rule=:a %{"name":"numbers", "type":"repeat", "parser":[ {"name":"n1", "type":"number"}, {"type":"literal", "text":":"}, {"name":"n2", "type":"number"} ], "while": { "type":"alternative", "parser": [ {"type":"literal", "text":", "}, {"type":"literal", "text":","} ] } }% b %w:word% ' execute 'a 1:2, 3:4,5:6, 7:8 b test' assert_output_json_eq '{ "w": "test", "numbers": [ { "n2": "2", "n1": "1" }, { "n2": "4", "n1": "3" }, { "n2": "6", "n1": "5" }, { "n2": "8", "n1": "7" } ] }' cleanup_tmp_files liblognorm-2.0.8/tests/rule_last_str_long.sh000077500000000000000000000012511511425433100212650ustar00rootroot00000000000000#!/bin/bash . $srcdir/exec.sh no_solaris10 test_def $0 "multiple formats including string (see also: rule_last_str_short.sh)" add_rule 'version=2' add_rule 'rule=:%string:string%' add_rule 'rule=:before %string:string%' add_rule 'rule=:%string:string% after' add_rule 'rule=:before %string:string% after' add_rule 'rule=:before %string:string% middle %string:string%' execute 'string' execute 'before string' execute 'string after' execute 'before string after' execute 'before string middle string' assert_output_json_eq '{"string": "string" }' '{"string": "string" }''{"string": "string" }''{"string": "string" }''{"string": "string", "string": "string" }' cleanup_tmp_files liblognorm-2.0.8/tests/rule_last_str_short.sh000077500000000000000000000003671511425433100214740ustar00rootroot00000000000000#!/bin/bash . $srcdir/exec.sh test_def $0 "string being last in a rule (see also: rule_last_str_long.sh)" add_rule 'version=2' add_rule 'rule=:%string:string%' execute 'string' assert_output_json_eq '{"string": "string" }' cleanup_tmp_files liblognorm-2.0.8/tests/runaway_rule.sh000077500000000000000000000011741511425433100201050ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 # Note that this test produces an error message, as it encounters the # runaway rule. This is OK and actually must happen. The prime point # of the test is that it correctly loads the second rule, which # would otherwise be consumed by the runaway rule. . $srcdir/exec.sh test_def $0 "runaway rule (unmatched percent signs)" reset_rules add_rule 'version=2' add_rule 'rule=:test %f1:word unmatched percent' add_rule 'rule=:%field:word%' execute 'data' assert_output_json_eq '{"field": "data"}' cleanup_tmp_files liblognorm-2.0.8/tests/runaway_rule_comment.sh000077500000000000000000000012321511425433100216220ustar00rootroot00000000000000#!/bin/bash # added 2015-09-16 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 # Note that this test produces an error message, as it encounters the # runaway rule. This is OK and actually must happen. The prime point # of the test is that it correctly loads the second rule, which # would otherwise be consumed by the runaway rule. . $srcdir/exec.sh test_def $0 "runaway rule with comment lines (v2)" reset_rules add_rule 'version=2' add_rule 'rule=:test %f1:word unmatched percent' add_rule '' add_rule '#comment' add_rule 'rule=:%field:word%' execute 'data' assert_output_json_eq '{"field": "data"}' cleanup_tmp_files liblognorm-2.0.8/tests/runaway_rule_comment_v1.sh000077500000000000000000000012051511425433100222300ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 # Note that this test produces an error message, as it encounters the # runaway rule. This is OK and actually must happen. The prime point # of the test is that it correctly loads the second rule, which # would otherwise be consumed by the runaway rule. . $srcdir/exec.sh test_def $0 "runaway rule with comment lines (v1)" reset_rules add_rule 'rule=:test %f1:word unmatched percent' add_rule '' add_rule '#comment' add_rule 'rule=:%field:word%' execute 'data' assert_output_json_eq '{"field": "data"}' cleanup_tmp_files liblognorm-2.0.8/tests/runaway_rule_v1.sh000077500000000000000000000011621511425433100205100ustar00rootroot00000000000000#!/bin/bash # added 2015-05-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 # Note that this test produces an error message, as it encounters the # runaway rule. This is OK and actually must happen. The prime point # of the test is that it correctly loads the second rule, which # would otherwise be consumed by the runaway rule. . $srcdir/exec.sh test_def $0 "runaway rule (unmatched percent signs) v1 version" reset_rules add_rule 'rule=:test %f1:word unmatched percent' add_rule 'rule=:%field:word%' execute 'data' assert_output_json_eq '{"field": "data"}' cleanup_tmp_files liblognorm-2.0.8/tests/seq_simple.sh000077500000000000000000000006771511425433100175400ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple sequence (array) syntax" add_rule 'version=2' add_rule 'rule=:a %[{"name":"num", "type":"number"}, {"name":"-", "type":"literal", "text":":"}, {"name":"hex", "type":"hexnumber"}]% b' execute 'a 4711:0x4712 b' assert_output_json_eq '{ "hex": "0x4712", "num": "4711" }' cleanup_tmp_files liblognorm-2.0.8/tests/strict_prefix_actual_sample1.sh000077500000000000000000000016211511425433100232250ustar00rootroot00000000000000#!/bin/bash # added 2015-11-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "one rule is strict prefix of a longer one" add_rule 'version=2' add_rule 'prefix=%timestamp:date-rfc3164% %hostname:word% BL-WLC01: *%programname:char-to:\x3a%: %timestamp:date-rfc3164%.%fracsec:number%:' add_rule 'rule=wifi: #LOG-3-Q_IND: webauth_redirect.c:1238 read error on server socket, errno=131[...It occurred %count:number% times.!]' add_rule 'rule=wifi: #LOG-3-Q_IND: webauth_redirect.c:1238 read error on server socket, errno=131' execute 'Sep 28 23:53:19 192.168.123.99 BL-WLC01: *dtlArpTask: Sep 28 23:53:19.614: #LOG-3-Q_IND: webauth_redirect.c:1238 read error on server socket, errno=131' assert_output_json_eq '{ "fracsec": "614", "timestamp": "Sep 28 23:53:19", "programname": "dtlArpTask", "hostname": "192.168.123.99" }' cleanup_tmp_files liblognorm-2.0.8/tests/strict_prefix_matching_1.sh000077500000000000000000000007331511425433100223470ustar00rootroot00000000000000#!/bin/bash # added 2015-11-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "one rule is strict prefix of a longer one" add_rule 'version=2' add_rule 'rule=:a word %w1:word%' add_rule 'rule=:a word %w1:word% another word %w2:word%' execute 'a word w1 another word w2' assert_output_json_eq '{ "w2": "w2", "w1": "w1" }' execute 'a word w1' assert_output_json_eq '{ "w1": "w1" }' cleanup_tmp_files liblognorm-2.0.8/tests/strict_prefix_matching_2.sh000077500000000000000000000011261511425433100223450ustar00rootroot00000000000000#!/bin/bash # added 2015-11-05 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "named literal compaction" add_rule 'version=2' add_rule 'rule=:a word %w1:word% %l1:literal{"text":"l"}% b' add_rule 'rule=:a word %w1:word% %l2:literal{"text":"l2"}% b' add_rule 'rule=:a word %w1:word% l3 b' execute 'a word w1 l b' assert_output_json_eq '{ "l1": "l", "w1": "w1" }' execute 'a word w1 l2 b' assert_output_json_eq '{ "l2": "l2", "w1": "w1" }' execute 'a word w1 l3 b' assert_output_json_eq '{ "w1": "w1" }' cleanup_tmp_files liblognorm-2.0.8/tests/string_rb_simple.sh000077500000000000000000000003661511425433100207340ustar00rootroot00000000000000#!/bin/bash # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple rulebase via string" execute_with_string 'rule=:%w:word%' 'test' assert_output_json_eq '{ "w": "test" }' cleanup_tmp_files liblognorm-2.0.8/tests/string_rb_simple_2_lines.sh000077500000000000000000000011751511425433100223460ustar00rootroot00000000000000#!/bin/bash # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple rulebase via string" execute_with_string 'rule=:%w:word% rule=:%n:number%' 'test' assert_output_json_eq '{ "w": "test" }' execute_with_string 'rule=:%w:word% rule=:%n:number%' '2' assert_output_json_eq '{ "n": "2" }' #This is a correct word... execute_with_string 'rule=:%w:word% rule=:%n:number%' '2.3' assert_output_json_eq '{ "w": "2.3" }' #check error case execute_with_string 'rule=:%w:word% rule=:%n:number%' '2 3' assert_output_json_eq '{ "originalmsg": "2 3", "unparsed-data": " 3" }' cleanup_tmp_files liblognorm-2.0.8/tests/user_test.c000066400000000000000000000012051511425433100172050ustar00rootroot00000000000000#include "config.h" #include #include "liblognorm.h" #include "v1_liblognorm.h" int main() { const char* str = "foo says hello!"; json_object *obj, *from, *msg; obj = from = msg = NULL; ln_ctx ctx = ln_initCtx(); int ret = 1; ln_v1_loadSample(ctx, "rule=:%from:word% says %msg:word%"); if (ln_v1_normalize(ctx, str, strlen(str), &obj) == 0) { json_object_object_get_ex(obj, "from", &from); json_object_object_get_ex(obj, "msg", &msg); ret = strcmp(json_object_get_string(from), "foo") || strcmp(json_object_get_string(msg), "hello!"); } if (obj != NULL) json_object_put(obj); ln_exitCtx(ctx); return ret; } liblognorm-2.0.8/tests/usrdef_actual1.sh000077500000000000000000000022361511425433100202720ustar00rootroot00000000000000#!/bin/bash # added 2015-10-30 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "an actual test case for user-defined types" add_rule 'version=2' add_rule 'type=@endpid:%{"type":"alternative","parser":[ {"type": "literal", "text":"]"},{"type": "literal", "text":"]:"} ] }%' add_rule 'type=@AUTOTYPE1:%iface:char-to:/%/%ip:ipv4%(%port:number%)' add_rule 'type=@AUTOTYPE1:%iface:char-to:\x3a%\x3a%ip:ipv4%/%port:number%' add_rule 'type=@AUTOTYPE1:%iface:char-to:\x3a%\x3a%ip:ipv4%' add_rule 'rule=:a pid[%pid:number%%-:@endpid% b' add_rule 'rule=:a iface %.:@AUTOTYPE1% b' execute 'a pid[4711] b' assert_output_json_eq '{ "pid": "4711" }' # the next text needs priority assignment #execute 'a pid[4712]: b' #assert_output_json_eq '{ "pid": "4712" }' execute 'a iface inside:10.0.0.1 b' assert_output_json_eq '{ "ip": "10.0.0.1", "iface": "inside" }' execute 'a iface inside:10.0.0.1/514 b' assert_output_json_eq '{ "port": "514", "ip": "10.0.0.1", "iface": "inside" }' execute 'a iface inside/10.0.0.1(514) b' assert_output_json_eq '{ "port": "514", "ip": "10.0.0.1", "iface": "inside" }' cleanup_tmp_files liblognorm-2.0.8/tests/usrdef_ipaddr.sh000077500000000000000000000011101511425433100201710ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple user-defined type" add_rule 'version=2' add_rule 'type=@IPaddr:%ip:ipv4%' add_rule 'type=@IPaddr:%ip:ipv6%' add_rule 'rule=:an ip address %.:@IPaddr%' execute 'an ip address 10.0.0.1' assert_output_json_eq '{ "ip": "10.0.0.1" }' execute 'an ip address 127::1' assert_output_json_eq '{ "ip": "127::1" }' execute 'an ip address 2001:DB8:0:1::10:1FF' assert_output_json_eq '{ "ip": "2001:DB8:0:1::10:1FF" }' cleanup_tmp_files liblognorm-2.0.8/tests/usrdef_ipaddr_dotdot.sh000077500000000000000000000011211511425433100215500ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "user-defined type with '..' name" add_rule 'version=2' add_rule 'type=@IPaddr:%..:ipv4%' add_rule 'type=@IPaddr:%..:ipv6%' add_rule 'rule=:an ip address %ip:@IPaddr%' execute 'an ip address 10.0.0.1' assert_output_json_eq '{ "ip": "10.0.0.1" }' execute 'an ip address 127::1' assert_output_json_eq '{ "ip": "127::1" }' execute 'an ip address 2001:DB8:0:1::10:1FF' assert_output_json_eq '{ "ip": "2001:DB8:0:1::10:1FF" }' cleanup_tmp_files liblognorm-2.0.8/tests/usrdef_ipaddr_dotdot2.sh000077500000000000000000000013041511425433100216350ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "user-defined type with '..' name embedded in other fields" add_rule 'version=2' add_rule 'type=@IPaddr:%..:ipv4%' add_rule 'type=@IPaddr:%..:ipv6%' add_rule 'rule=:a word %w1:word% an ip address %ip:@IPaddr% another word %w2:word%' execute 'a word word1 an ip address 10.0.0.1 another word word2' assert_output_json_eq '{ "w2": "word2", "ip": "10.0.0.1", "w1": "word1" }' execute 'a word word1 an ip address 2001:DB8:0:1::10:1FF another word word2' assert_output_json_eq '{ "w2": "word2", "ip": "2001:DB8:0:1::10:1FF", "w1": "word1" }' cleanup_tmp_files liblognorm-2.0.8/tests/usrdef_ipaddr_dotdot3.sh000077500000000000000000000020271511425433100216410ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "user-defined type with '..' name embedded in other fields" add_rule 'version=2' add_rule 'type=@IPaddr:%..:ipv4%' add_rule 'type=@IPaddr:%..:ipv6%' add_rule 'type=@ipOrNumber:%..:@IPaddr{"priority":"1000"}%' add_rule 'type=@ipOrNumber:%..:number%' #add_rule 'type=@ipOrNumber:%..:@IPaddr%' # if we enable this instead of the above, the test would break add_rule 'rule=:a word %w1:word% an ip address %ip:@ipOrNumber% another word %w2:word%' execute 'a word word1 an ip address 10.0.0.1 another word word2' assert_output_json_eq '{ "w2": "word2", "ip": "10.0.0.1", "w1": "word1" }' execute 'a word word1 an ip address 2001:DB8:0:1::10:1FF another word word2' assert_output_json_eq '{ "w2": "word2", "ip": "2001:DB8:0:1::10:1FF", "w1": "word1" }' execute 'a word word1 an ip address 111 another word word2' assert_output_json_eq '{ "w2": "word2", "ip": "111", "w1": "word1" }' cleanup_tmp_files liblognorm-2.0.8/tests/usrdef_nested_segfault.sh000077500000000000000000000010461511425433100221120ustar00rootroot00000000000000#!/bin/bash # added 2018-04-07 by Vincent Tondellier # based on usrdef_twotypes.sh # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "nested user-defined types" add_rule 'version=2' add_rule 'type=@hex-byte:%..:hexnumber{"maxval": "255"}%' add_rule 'type=@two-hex-bytes:%f1:@hex-byte% %f2:@hex-byte%' add_rule 'type=@unused:stop' add_rule 'rule=:two bytes %.:@two-hex-bytes% %-:@unused%' execute 'two bytes 0xff 0x16 stop' assert_output_json_eq '{ "f1": "0xff", "f2": "0x16" }' cleanup_tmp_files liblognorm-2.0.8/tests/usrdef_simple.sh000077500000000000000000000007321511425433100202300ustar00rootroot00000000000000#!/bin/bash # added 2015-07-22 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "simple user-defined type" add_rule 'version=2' add_rule 'type=@hex-byte:%f1:hexnumber{"maxval": "255"}%' add_rule 'rule=:a word %w1:word% a byte % .:@hex-byte % another word %w2:word%' execute 'a word w1 a byte 0xff another word w2' assert_output_json_eq '{ "w2": "w2", "f1": "0xff", "w1": "w1" }' cleanup_tmp_files liblognorm-2.0.8/tests/usrdef_two.sh000077500000000000000000000012001511425433100175370ustar00rootroot00000000000000#!/bin/bash # added 2015-10-30 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "user-defined type with two alternatives" add_rule 'version=2' add_rule 'type=@hex-byte:%f1:hexnumber{"maxval": "255"}%' add_rule 'type=@hex-byte:%f1:word%' add_rule 'rule=:a word %w1:word% a byte % .:@hex-byte % another word %w2:word%' execute 'a word w1 a byte 0xff another word w2' assert_output_json_eq '{ "w2": "w2", "f1": "0xff", "w1": "w1" }' execute 'a word w1 a byte TEST another word w2' assert_output_json_eq '{ "w2": "w2", "f1": "TEST", "w1": "w1" }' cleanup_tmp_files liblognorm-2.0.8/tests/usrdef_twotypes.sh000077500000000000000000000010031511425433100206250ustar00rootroot00000000000000#!/bin/bash # added 2015-10-30 by Rainer Gerhards # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh test_def $0 "two user-defined types" add_rule 'version=2' add_rule 'type=@hex-byte:%f1:hexnumber{"maxval": "255"}%' add_rule 'type=@word-type:%w1:word%' add_rule 'rule=:a word %.:@word-type% a byte % .:@hex-byte % another word %w2:word%' execute 'a word w1 a byte 0xff another word w2' assert_output_json_eq '{ "w2": "w2", "f1": "0xff", "w1": "w1" }' cleanup_tmp_files liblognorm-2.0.8/tests/very_long_logline.sh000077500000000000000000000006731511425433100211100ustar00rootroot00000000000000#!/bin/bash # added 2015-09-21 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh msg="foo" for i in $(seq 1 10); do msg="${msg},${msg},abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ${i}" done test_def $0 "float field" add_rule 'rule=:%line:rest%' execute $msg assert_output_json_eq "{\"line\": \"$msg\"}" cleanup_tmp_files liblognorm-2.0.8/tests/very_long_logline_jsoncnf.sh000077500000000000000000000007451511425433100226300ustar00rootroot00000000000000#!/bin/bash # added 2015-09-21 by singh.janmejay # This file is part of the liblognorm project, released under ASL 2.0 . $srcdir/exec.sh msg="foo" for i in $(seq 1 10); do msg="${msg},${msg},abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ${i}" done test_def $0 "float field" add_rule 'version=2' add_rule 'rule=:%{"name":"line", "type":"rest"}%' execute $msg assert_output_json_eq "{\"line\": \"$msg\"}" cleanup_tmp_files liblognorm-2.0.8/tools/000077500000000000000000000000001511425433100150245ustar00rootroot00000000000000liblognorm-2.0.8/tools/Makefile.am000066400000000000000000000005111511425433100170550ustar00rootroot00000000000000# Note: slsa is not yet functional with the v2 engine! #bin_PROGRAMS = slsa #slsa_SOURCES = slsa.c syntaxes.c ../src/parser.c #slsa_CPPFLAGS = -I$(top_srcdir)/src $(JSON_C_CFLAGS) #slsa_LDADD = $(JSON_C_LIBS) $(LIBLOGNORM_LIBS) $(LIBESTR_LIBS) #slsa_DEPENDENCIES = ../src/liblognorm.la EXTRA_DIST=logrecord.h #include_HEADERS= liblognorm-2.0.8/tools/lablog000066400000000000000000000330711511425433100162130ustar00rootroot00000000000000This is a log of findings during development of the slsa heuristic Terms ----- log message a string of printable characters, delimited by the operating system line terminator. word a substring inside a log message that is delimited by specific delimiters, usually whitespace [this definition may need to be changed] subword a sequence inside a word that is not delimited by the usual word delimiters MSA multiple sequence alignment (like used in bioinformatics) motif a substring inside a log message that is frequently being used and has a specific syntax and semantic (e.g. an IPv4 address). The term is based on the idea of "sequence motif" in bioinformatics. parser (also "motif parser") extracts actual data matching a given motif from a log message. Open Problems ------------- [P00001] How to detect TAG? syslog TAG does not work well with common prefix/suffix detection. The problem is that e.g. PIX tags have integers, which are detected and Linux logs have process IDs. The PIX integers are actually part of the constant text, whereas in Linux it is fine to detect them as variable part. This problem becomes even more problematic with more agressive word delimition (see [I00002]). [P00002] subword detection with optional parts Sometimes, subwords may contain optional parts, so not all words may contain the common delimiter (e.g. Cisco PIX). In this case, subword detection is not triggered. Open TODO Items --------------- Note: TODO items regarding motif parsers can also be found at https://github.com/rsyslog/liblognorm/issues?utf8=%E2%9C%93&q=is%3Aissue+label%3A%22motif+parser%22 [T00001] - Dictionary motif We need to add dictionary motifs, where one can e.g. query users or common words. It may make sense to ship some "common-word dictionaries". Common word (sets) are like ("TCP", "UDP", "ICMP", ...) which are often found in firewall messages. With such words, subword detection does not prove to be helpful, because it detects a common suffix like [["TC", "UD"] "P"]. [T00002] - URL motif Logs often contain URLs (and file pathes), just like this: 193.45.10.119:/techsupp/css/default.css We need to find a proper way to identify them. This motif is probably very broad and hard to specify, especially when thinking about multiple platforms. [T00003] - name=value motif name=value is very common and justifed to be treated as motif. [T00004] - quote characters in n=v pairs We need to find out which quote characters happen to be present in especially values of a wide variety of Linux logs (at least the typical ones). This can be used to better describe the n=v motif. Recommended further testing --------------------------- This section lists some tests that would be good to conduct as part of the project. However, it is NOT thought that these experiments are vital for the results. - test slsa on iptables logs with the regular n=v parser We want to see here if there is a way to still properly detect the structure. Maybe we need to shuffle the "nondetected" word elements. - test slsa without cisco-interface-spec parser Same question as above, can the generic heuristic sufficiently well detect structure? The core goal of these tests is to find ways to improve the slsa algorithm. Indicators in first column of actual log ---------------------------------------- - lesson learned, general info ! idea + result of idea Ideas are referenced by [Innnnn] at start of log entry Actual Log 09:55:01 ---------- 2015-04-14 - Cisco ASA has ample slighly different formats for specifying interface addresses (the faddr, gaddr, laddr parts of message). It looks like the log strings are written directly in code, at least we have a lot of small inconsistencies which suggest so. This seems to makes it impractical to generate a single motif for this type. - Based on visual review, it looks like pattern detection on a subword-level may make sense (e.g. detect IP address, then you'll find a common slash, then a port number). This boils down to doing a full MSA on the word level, which I tried to avoid for performance reasons. ! [I00001] We may do an subword alignment on some common delimiters like slash, comma, colon etc. This can probably be done much faster than a full MSA. ! [I00002] We may also experiment with additional "word-delimiters", optionally enable those from the same set as [I00001]. When this is done, we need to make sure the delimiter can be stored inside the rulebase (an additional step to be taken only of this idea proves to be useful). 2015-04-15 + [I0002] First rought tests indicate that additional word delimiters seem to work well, at least on Cisco messages. A problem is that TAG values are now intermangled, as a TAG typically is "name:" and the colon is now a word delimiter. This leads to all TAGs becoming one big group, and only after the colon differentiation occurs. That, however, is too late, especially for things like fixed-width structured formats (e.g. squid). This handling of the TAG is a big problem, because the TAG is usually the primary identifier for a message class. So it should be preserved as unique. The same problem shows up when processing Linux logs. The TAG becomes effectively unusable as a way to identify the message. I also have problems interpreting postfix logs correctly, if "[]" is part of the delimiter set. I have not tried yet to trace the root cause down, as the approach in general seems to be problematic in regard to TAG. I suspect that this "postfix problem" is actually related to the TAG. Looks like [P00001] must be looked at with priority in order to continue with useful analysis. Another problem that manual review brings up is that colon is often used e.g. in time formats ("12:04:11"). If we use colon as word delimiter, we are no longer able to detect such time formats. This suggest that more agressive word delimition is probably not a good thing to have. However, it looks like we could do the same during subword detection stage without big problems. Ideally, this would be a kind of alignment like proposed in [I00001]. 2015-04-16 + [I00001][I00002] Tried subword detection with colon and slash as delimiters. Works fine on Cisco logs and does not have any of the problem we had with [I0002] (as described yesterday). One problem is if we have something like this: 7l: connection {5} 8l: %posint% {5} 9l: for {5} 10l: outside:192.168.66.144/80 {3} 11l: (192.168.66.144/80) {3} 12l: to {3} 13l: inside:192.168.12.154/56839 14l: (217.17.249.222/56839) [nterm 1] 13l: inside:192.168.12.154/56842 14l: (217.17.249.222/56842) [nterm 1] 13l: inside:192.168.12.154/56845 14l: (217.17.249.222/31575) [nterm 1] The algo will explode the lower "inside:" parts individually because each node is of course processed individually. So the result will be: 7l: connection {5} 8l: %posint% {5} 9l: for {5} 10l: outside {subword} {3} 11l: : {subword} 12l: %ipv4% {subword} 13l: / {subword} 14l: %posint% {subword} 15l: (192.168.66.144 / 80) to {subword} {3} 16l: inside {subword} 17l: : {subword} 18l: %ipv4% {subword} 19l: / {subword} 20l: %posint% {subword} 21l: (217.17.249.222 / 56839) {subword} 16l: inside {subword} 17l: : {subword} 18l: %ipv4% {subword} 19l: / {subword} 20l: %posint% {subword} 21l: (6.79.249.222 / 56842) {subword} 16l: inside {subword} 17l: : {subword} 18l: %ipv4% {subword} 19l: / {subword} 20l: %posint% {subword} 21l: (217.17.249.222 / 31575) {subword} However, we expect that this will not affect the final rule generation. But needs to be proven. [I00003] Once we have split subwords, we may do another "alignment run" on the tree and check if we now can find additional things to combine. Needs to be seen. In any case, we need to split braces, which requires slight changes to the split algo. We also see that the subword split algo does not work properly if we have words of different formats. Cisco PIX, for example, has interface specifiers which may either be "IP" or "IP/PORT" (among others). In that case, the delimiter "/" is not detected as common delimiter and so subword detection is not triggered. Now tracked as [P00002]. - I had a frist try at using "=" as a subword delimiter. This works for name=value fields, but only if they are all in the same sequence. It looks like it is a much better idea, for real N=V type of log messages to generate an parser that works on the complete line (things like iptables) [T00003]. It may still make sense to have individual N=V parsers if these constructs are seen within otherwise non-structured messages. 2015-04-17 - as expected, the Linux kernel timestamp motif parser proved to be useful. 2015-04-29 - some interim entries are missing, as I was focussed on some other work, including support for some structured formats. - I added cee-syslog and pure JSON motifs for structured formats. As expected, this permits slsa to process log files that contain both structured and unstructured formats (a common use case) much more rapidly and also improved the correct detection. Note that pure JSON is seen for example in GELF, wheres cee-syslog is seen in ossec. Both are frequently used. - I also worked on a specifc motif parser for interface specifiers found in Cisco logs. This, too, improved detection, but at a later stage we may want to try if we can gain to similar results without a specific parser. For practical cases, however, a specific parser is much more user-friendly. - I have also continued to work on a specific Name=Value (N=V) parser [T00003]. A first test some days ago showed that iptables needs a special parser because it also has name "flags", that is a field without a value and WITHOUT an equal sign (e.g. "DF"). This is not something we want in the regular n=v motif as it would match much to broadly. For iptables, we can do other restrictions (e.g. names need to consist of uppercase letters A..Z). So an iptable motif would not match too broad. Unfortunately, iptables is an example of where a specialised parser is needed for a single application. At least from the slsa point of view, this is bad, because it shows that the tool has limited potential. From a practical perspective, that iptables motif is definitely frequent enough to justify the (small) effort to implement a specific parser. - on the n=v parser: it turns out to be somewhat problematic to find good sets of characters permitted in name and value. Value is not so much a problem for quoted strings, but unquoted strings are very common. For example "session opened for user root by (uid=0)" will lead to "session opened for user root by (%name-value-list%" This shows a) the we must not insist on whitespace before the name, as quoted forms (like the brace here) are common b) but the terminating brace is treated as part of the value if we only use whitespace delimition Especially b) is a bit hard of a problem, because we may also see things like "name=(value)", so it seems we cannot forbid braces in values. HOWEVER, we may actually forbid them if inside the string and not being escaped - so think of them as just different types of quoting, which they are most probably like. This should be verified by experiments. --> [T00004] 2015-05-03 - did some experiments with the v2-iptables parser I had newly written. It turned out that the parser is matching too broad (e.g a single upper case latter will qualify as an iptables entry, because it looks like a single flag value). While this is fine with a manually crafted rulebase, it doesn't work with tools like slsa, which try to detect what the motif is. Also, it may cause misdetection even in the manually crafted case. So this is a very good proof that motif parsers (and thus the motifs) need to be very well defined and as specific as possible. We will sort this out with the idea that an iptables entry always has more than a single word. + fixing the v2-iptables parser as proposed above did solve the issue with misdetection liblognorm-2.0.8/tools/logrecord.h000066400000000000000000000032261511425433100171600ustar00rootroot00000000000000/* The (in-memory) format of a log record. * * A log record is sequence of nodes of different syntaxes. A log * record is described by a pointer to its root node. * The most important node type is literal text, which is always * assumed if no other syntax is detected. A full list of syntaxes * can be found below. * * Copyright 2015 Rainer Gerhards * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef LOGRECORD_H_INCLUDED #define LOGRECORD_H_INCLUDED #include /* log record node syntaxes * This "enumeration" starts at 0 and increments for each new * syntax. Note that we do not use an enum type so that we can * streamline the in-memory representation. For large sets of * log records to be held in main memory, this is important. */ #define LRN_SYNTAX_LITERAL_TEXT 0 #define LRN_SYNTAX_IPV4 1 #define LRN_SYNTAX_INT_POSITIVE 2 #define LRN_SYNTAX_DATE_RFC3164 3 struct logrec_node { struct logrec_node *next; /* NULL: end of record */ int8_t ntype; union { char *ltext; /* the literal text */ int64_t number; /* all integer types */ } val; }; typedef struct logrec_node logrecord_t; #endif /* ifndef LOGRECORD_H_INCLUDED */ liblognorm-2.0.8/tools/slsa.c000066400000000000000000001040461511425433100161370ustar00rootroot00000000000000/* simple log structure analyzer (slsa) * * This is a heuristic to mine log structure. * * Copyright 2015 Rainer Gerhards * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* Learnings (mostly from things that failed) * ------------------------------------------ * - if we detect IP (and similar things) too early in the process, we * get wrong detections (e.g. for reverse DNS entries) * - if we detect IP adresses during value collapsing, we get unacceptable * runtime and memory requirements, as a huge number of different actual * values needs to be stored. * ->current solution is to detect them after tokenization but before * adding the the structure tree. * * - if we split some known syntaxes (like cisco ip/port) into it's parts, * they may get improperly detected in regard to subwords. This especially * happens if there are few servers (or few destination ports), where parts * of the IP (or the port) is considered to be a common prefix, which then * may no longer be properly detected. As it looks, it pays to really have a * much richer set of special parsers, and let the common pre/suffix * detection only be used in rare cases where we do not have a parser. * Invocation may even be treated as the need to check for a new parser. * * Along the same lines, common "command words" (like "udp", "tcp") may * also be detected by a specific parser, because otherwise they tend to have * common pre/suffixes (e.g. {"ud","tc"},"p"). This could be done via a * dictionary lookup (bsearch, later to become a single state machine?). * * Along these lines, we probably need parses who return *multiple* values, * e.g. the IP address and port. This requires that the field name has a * common prefix. A solution is to return JSON where the node element is * named like the field name, and the members have parser-specific names. */ #include "config.h" #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include "json_compatibility.h" #include "liblognorm.h" #include "internal.h" #include "parser.h" #include "syntaxes.h" #define MAXLINE 32*1024 struct wordflags { unsigned isSubword : 1, isSpecial : 1; /* indicates this is a special parser */ }; struct wordinfo { char *word; int occurs; struct wordflags flags; }; struct logrec_node { struct logrec_node *parent; struct logrec_node *sibling; struct logrec_node *child; /* NULL: end of record */ int nterm; /* number of times this was the terminal node */ int8_t ntype; union { char *ltext; /* the literal text */ int64_t number; /* all integer types */ } val; /* the addtl value structure is dumb, and needs to * be replaced with something much better when the initial * experiments work out. But good enough for now. * (bsearch table?) */ int nwords; /* size of table */ struct wordinfo **words; }; typedef struct logrec_node logrec_node_t; logrec_node_t *root = NULL; typedef struct rule_table_etry rule_table_etry_t; typedef struct rule_table rule_table_t; struct rule_table { int maxEtry; /* max # entries that fit into table */ int nxtEtry; /* next free entry */ rule_table_etry_t **entries; }; struct rule_table_etry { int ntimes; char *rule; }; #define RULE_TABLE_GROWTH 512 /* number of entries rule table grows when too small */ /* command line options */ static int displayProgress = 0; /* display progress indicators */ static int optPrintTree = 0; /* disply internal tree for debugging purposes */ static int optPrintDebugOutput = 0; static int optSortMultivalues = 1; /* forward definitions */ void wordDetectSyntax(struct wordinfo *const __restrict__ wi, const size_t wordlen, const int); void treePrint(logrec_node_t *node, const int level); static void reportProgress(const char *const label); /* param word may be NULL, if word is not yet known */ static struct wordinfo * wordinfoNew(char *const word) { struct wordinfo *const wi = calloc(1, sizeof(struct wordinfo)); if(wi == NULL) { perror("slsa: malloc error struct wordinfo"); exit(1); } wi->word = word; wi->occurs = 1; return wi; } static void wordinfoDelete(struct wordinfo *const wi) { //printf("free %p: %s\n", wi->word, wi->word); free(wi->word); free(wi); } static rule_table_t * ruleTableCreate(void) { rule_table_t *const rt = calloc(1, sizeof(rule_table_t)); if(rt == NULL) { perror("slsa: malloc error ruletable_create"); exit(1); } rt->nxtEtry = 0; rt->maxEtry = 0; return rt; } static rule_table_etry_t * ruleTableEtryCreate(rule_table_t *const __restrict__ rt) { if(rt->nxtEtry == rt->maxEtry) { const int newMax = rt->maxEtry + RULE_TABLE_GROWTH; rt->entries = realloc(rt->entries, newMax * sizeof(rule_table_t)); if(rt->entries == NULL) { perror("slsa: malloc error ruletable_create"); exit(1); } rt->maxEtry = newMax; } const int etry = rt->nxtEtry; rt->nxtEtry++; rt->entries[etry] = calloc(1, sizeof(rule_table_etry_t)); if(rt->entries[etry] == NULL) { perror("slsa: malloc error entry ruletable_create"); exit(1); } return rt->entries[etry]; } static void ruleTablePrint(rule_table_t *const __restrict__ rt) { for(int i = 0 ; i < rt->nxtEtry ; ++i) { reportProgress("rule table print"); printf("%6d times: %s\n", rt->entries[i]->ntimes, rt->entries[i]->rule); } } static void ruleTableDestroy(rule_table_t *const __restrict__ rt) { for(int i = 0 ; i < rt->nxtEtry ; ++i) { free((void*)rt->entries[i]->rule); free((void*)rt->entries[i]); } free((void*) rt->entries); free((void*) rt); } /* function to quicksort rule table */ static int qs_comp_rt_etry(const void *v1, const void *v2) { const rule_table_etry_t *const e1 = *((rule_table_etry_t **) v1); const rule_table_etry_t *const e2 = *((rule_table_etry_t **) v2); return -(e1->ntimes - e2->ntimes); /* sort descending! */ } /* a stack to keep track of detected (sub)words that * need to be processed in the future. */ #define SIZE_WORDSTACK 8 static struct wordinfo* wordStack[SIZE_WORDSTACK]; static int wordstackPtr = -1; /* empty */ static void wordstackPush(struct wordinfo *const wi) { ++wordstackPtr; if(wordstackPtr >= SIZE_WORDSTACK) { fprintf(stderr, "slsa: wordstack too small\n"); exit(1); } wordStack[wordstackPtr] = wi; } /* returns wordinfo or NULL, if stack is empty */ static struct wordinfo * wordstackPop(void) { return (wordstackPtr < 0) ? NULL : wordStack[wordstackPtr--]; } static void reportProgress(const char *const label) { static unsigned cnt = 0; static const char *lastlabel = NULL; if(!displayProgress) return; if(lastlabel == NULL) lastlabel = strdup(label); if(label == NULL || strcmp(label, lastlabel)) { fprintf(stderr, "\r%s: %u - done\n", lastlabel, cnt); cnt = 0; free((void*)lastlabel); lastlabel = (label == NULL) ? NULL : strdup(label); } else { if(++cnt % 100 == 0) fprintf(stderr, "\r%s: %u", label, cnt); } } static int qs_compmi(const void *v1, const void *v2) { const struct wordinfo *const w1 = *((struct wordinfo**) v1); const struct wordinfo *const w2 = *((struct wordinfo**) v2); return strcmp(w1->word, w2->word); } #if 0 /* we don't need this at the moment, but want to preserve it * in case we can need it again. */ static int bs_compmi(const void *key, const void *mem) { const struct wordinfo *const wi = *((struct wordinfo**) mem); return strcmp((char*)key, wi->word); } #endif /* add an additional word to existing node */ void logrec_addWord(logrec_node_t *const __restrict__ node, struct wordinfo *const __restrict__ wi) { /* TODO: here we can insert at the right spot, which makes * bsearch() usable [in the caller, later...] */ /* TODO: alloc in larger increments */ int newnwords = node->nwords + 1; node->words = realloc(node->words, sizeof(struct wordinfo *)*newnwords); wi->occurs = 1; /* TODO: layer violation, do in other function */ node->words[node->nwords] = wi; node->nwords = newnwords; #if 0 if(node->nwords > 1) { qsort(node->words, node->nwords, sizeof(struct wordinfo), qs_compmi); // TODO upgrade } #endif } logrec_node_t * logrec_newNode(struct wordinfo *const wi, struct logrec_node *const parent) { logrec_node_t *const node = calloc(1, sizeof(logrec_node_t)); node->parent = parent; node->child = NULL; node->val.ltext = NULL; if(wi != NULL) logrec_addWord(node, wi); return node; } void logrec_delNode(logrec_node_t *const node) { for(int i = 0 ; i < node->nwords ; ++i) wordinfoDelete(node->words[i]); free(node->words); free(node->val.ltext); free(node); } /* returns ptr to existing struct wordinfo or NULL, if not found. */ static struct wordinfo * logrec_hasWord(logrec_node_t *const __restrict__ node, char *const __restrict__ word) { // return (struct wordinfo*) bsearch(word, node->words, node->nwords, sizeof(struct wordinfo), bs_compmi); /* alternative without need to have a binary array -- may make sense... */ int i; for(i = 0 ; i < node->nwords ; ++i) { if(!strcmp(node->words[i]->word, word)) break; } return (i < node->nwords) ? node->words[i] : NULL; } void printPrefixes(logrec_node_t *const __restrict__ node, const int lenPrefix, const int lenSuffix) { if(!optPrintDebugOutput) return; int i; const int maxwords = node->nwords > 5 ? 5 : node->nwords; printf("prefix %d, suffix %d\n", lenPrefix, lenSuffix); for(i = 0 ; i < maxwords ; ++i) { const char *const word = node->words[i]->word; const int lenWord = strlen(word); const int strtSuffix = lenWord - lenSuffix; int j; putchar('"'); for(j = 0 ; j < lenPrefix ; ++j) putchar(word[j]); printf("\" \""); for( ; j < strtSuffix ; ++j) putchar(word[j]); printf("\" \""); for( ; j < lenWord ; ++j) putchar(word[j]); printf("\"\n"); } } /* Squash duplicate values inside a tree node. This must only be run * after tree node values have been modified. */ static void squashDuplicateValues(logrec_node_t *node) { if(node->nwords == 1) return; /* sort and the remove the easy to find dupes */ qsort(node->words, node->nwords, sizeof(struct wordinfo*), qs_compmi); int iPrev = 0; int nsquashed = 0; for(int iNext = 1 ; iNext < node->nwords ; ++iNext) { if(!strcmp(node->words[iPrev]->word, node->words[iNext]->word)) { wordinfoDelete(node->words[iNext]); ++nsquashed; } else { /* OK, new word */ if(iPrev+1 == iNext) iPrev = iNext; else { node->words[iPrev]->occurs += iNext - iPrev - 1; ++iPrev; node->words[iPrev] = node->words[iNext]; } } } if(nsquashed) { node->nwords -= nsquashed; node->words = realloc(node->words, sizeof(struct wordinfo *)*node->nwords); } } /* Disjoin subwords based on a Delimiter. * When calling this function, it must be known that the Delimiter * is present in *all* words. * TODO: think about empty initial subwords (Delimiter in start position!). */ static void disjoinDelimiter(logrec_node_t *node, const char Delimiter) { /* first, create node pointers: * we need to update our original node in-place, because otherwise * we change the structure of the tree with a couple of * side effects. As we do not want this, the first subword must * be placed into the current node, and new nodes be * created for the other subwords. */ char delimword[2]; delimword[0] = Delimiter; delimword[1] = '\0'; struct wordinfo *const delim_wi = wordinfoNew(strdup(delimword)); delim_wi->flags.isSubword = 1; logrec_node_t *const delim_node = logrec_newNode(delim_wi, node); logrec_node_t *const tail_node = logrec_newNode(NULL, delim_node); delim_node->child = tail_node; tail_node->child = node->child; node->child = delim_node; delim_node->parent = node; if(tail_node->child != NULL) tail_node->child->parent = tail_node; /* now, do the actual split */ //printf("nodes setup, now doing actual split\n");fflush(stdout); char *prevword = NULL; for(int i = 0 ; i < node->nwords ; ++i) { struct wordinfo *wi; char *const delimptr = strchr(node->words[i]->word, Delimiter); //printf("splitting off tail %d of %d [%p]:'%s' [full: '%s']\n" //, i, node->nwords, delimptr+1, delimptr+1, node->words[i]->word);fflush(stdout); wi = wordinfoNew(strdup(delimptr+1)); wi->flags.isSubword = 1; wordDetectSyntax(wi, strlen(wi->word), 0); /* add new word only if not duplicate of previous (reduces dupes) */ if(prevword == NULL || strcmp(prevword, wi->word)) { logrec_addWord(tail_node, wi); prevword = wi->word; } else { wordinfoDelete(wi); } /* we can now do an in-place update of the old word ;) */ *delimptr = '\0'; node->words[i]->flags.isSubword = 1; wordDetectSyntax(node->words[i], strlen(node->words[i]->word), 0); #if 0 /* but we trim the memory */ const char *delword = node->words[i]->word; node->words[i]->word = strdup(delword); free((void*)delword); #endif } if(node->nwords > 1) { squashDuplicateValues(node); squashDuplicateValues(tail_node); } //printf("done disjonDelimiter\n");fflush(stdout); } /* Disjoin common prefixes and suffixes. This also triggers a * new syntax detection on the remaining variable part. */ static void disjoinCommon(logrec_node_t *node, const size_t lenPrefix, const size_t lenSuffix) { logrec_node_t *newnode; struct wordinfo *newwi; char *newword; char *word = node->words[0]->word; if(lenPrefix > 0) { /* we need to update our node in-place, because otherwise * we change the structure of the tree with a couple of * side effects. As we do not want this, the prefix must * be placed into the current node, and new nodes be * created for the other words. */ newword = malloc(lenPrefix+1); memcpy(newword, word, lenPrefix); newword[lenPrefix] = '\0'; newwi = wordinfoNew(newword); newwi->flags.isSubword = 1; newnode = logrec_newNode(newwi, node); newnode->words = node->words; newnode->nwords = node->nwords; node->nwords = 0; node->words = NULL; logrec_addWord(node, newwi); newnode->child = node->child; node->child = newnode; node->parent = newnode; /* switch node */ node = newnode; for(int i = 0 ; i < node->nwords ; ++i) memmove(node->words[i]->word, /* move includes \0 */ node->words[i]->word+lenPrefix, strlen(node->words[i]->word)-lenPrefix+1); } if(lenSuffix > 0) { const size_t lenword = strlen(word); size_t iSuffix = lenword-lenSuffix; newword = malloc(lenSuffix+1); memcpy(newword, word+iSuffix, lenSuffix+1); /* includes \0 */ newwi = wordinfoNew(newword); newwi->flags.isSubword = 1; newnode = logrec_newNode(newwi, node); newnode->child = node->child; if(newnode->child != NULL) newnode->child->parent = newnode; node->child = newnode; for(int i = 0 ; i < node->nwords ; ++i) { iSuffix = strlen(node->words[i]->word)-lenSuffix; node->words[i]->word[iSuffix] = '\0'; } } for(int i = 0 ; i < node->nwords ; ++i) { node->words[i]->flags.isSubword = 1; wordDetectSyntax(node->words[i], strlen(node->words[i]->word), 0); } // TODO: squash duplicates only if syntaxes were detected! squashDuplicateValues(node); } /* find a matching terminator inside a suffix, searchs only * within the suffix area. If found, lenPrefix and lenSuffix * are update and 1 is returned. Returns 0 if not found. * Helper to checkPrefixes. */ static int findMatchingTerm(const char *const __restrict__ word, const size_t lenWord, size_t potentialNewPrefix, int *const __restrict lenPrefix, int *const __restrict lenSuffix, const char term) { int newSuffix = -1; for(int i = 0 ; i < *lenSuffix ; ++i) if(word[lenWord-i-1] == term) { newSuffix = i+1; break; } if(newSuffix >= 0) { *lenSuffix = newSuffix; *lenPrefix = potentialNewPrefix; return 1; } return 0; } /* returns 1 if Delimiter is found in all words, 0 otherwise */ int checkCommonDelimiter(logrec_node_t *const __restrict__ node, const char Delimiter) { for(int i = 0 ; i < node->nwords ; ++i) { if(strlen(node->words[i]->word) < 2 || strchr(node->words[i]->word+1, Delimiter) == NULL) return 0; } return 1; } /* returns 1 if braces are found in all words, 0 otherwise */ int checkCommonBraces(logrec_node_t *const __restrict__ node, const char braceOpen, const char braceClose) { char *op; for(int i = 0 ; i < node->nwords ; ++i) { if(strlen(node->words[i]->word) < 2) return 0; if((op = strchr(node->words[i]->word+1, braceOpen)) == NULL) return 0; else if(strchr(op+1, braceClose) == NULL) return 0; } return 1; } /* check if there are common subword delimiters inside the values. If so, * use them to create subwords. Revalute the syntax if done so. */ void checkSubwords(logrec_node_t *const __restrict__ node) { //printf("checkSubwords checking node %p: %s\n", node, node->words[0]->word); if(checkCommonDelimiter(node, '/')) { disjoinDelimiter(node, '/'); } if(checkCommonDelimiter(node, ':')) { disjoinDelimiter(node, ':'); } if(checkCommonDelimiter(node, '=')) { disjoinDelimiter(node, '='); } #if 0 // this does not work, requies a seperate disjoin operation (4-parts) if(checkCommonBraces(node, '[', ']')) { disjoinDelimiter(node, '('); disjoinDelimiter(node->child->child, ')'); } #endif } /* check if there are common prefixes and suffixes and, if so, * extract them. */ void checkPrefixes(logrec_node_t *const __restrict__ node) { if(node->nwords == 1 || node->words[0]->flags.isSubword) return; int i; const char *const baseword = node->words[0]->word; const size_t lenBaseword = strlen(baseword); int lenPrefix = lenBaseword; int lenSuffix = lenBaseword; int shortestWord = INT_MAX; for(i = 1 ; i < node->nwords ; ++i) { int j; /* check prefix */ if(lenPrefix > 0) { for(j = 0 ; j < lenPrefix && node->words[i]->word[j] == baseword[j] ; ++j) ; /* EMPTY - just scan */ if(j < lenPrefix) lenPrefix = j; } /* check suffix */ if(lenSuffix > 0) { const int lenWord = strlen(node->words[i]->word); if(lenWord < shortestWord) shortestWord = lenWord; const int jmax = (lenWord < lenSuffix) ? lenWord : lenSuffix; for(j = 0 ; j < jmax && node->words[i]->word[lenWord-j-1] == baseword[lenBaseword-j-1] ; ++j) ; /* EMPTY - just scan */ if(j < lenSuffix) lenSuffix = j; } } if(lenPrefix+lenSuffix > shortestWord) /* can happen, e.g. if {"aaa","aaa"} */ lenSuffix = shortestWord - lenPrefix; /* to avoid false positives, we check for some common * field="xxx" syntaxes here. */ for(int j = lenPrefix-1 ; j >= 0 ; --j) { switch(baseword[j]) { case '"': if(findMatchingTerm(baseword, lenBaseword, j+1, &lenPrefix, &lenSuffix,'"')) goto done_prefixes; break; case '\'': if(findMatchingTerm(baseword, lenBaseword, j+1, &lenPrefix, &lenSuffix,'\'')) goto done_prefixes; break; case '[': if(findMatchingTerm(baseword, lenBaseword, j+1, &lenPrefix, &lenSuffix,']')) goto done_prefixes; break; case '(': if(findMatchingTerm(baseword, lenBaseword, j+1, &lenPrefix, &lenSuffix,')')) goto done_prefixes; break; case '<': if(findMatchingTerm(baseword, lenBaseword, j+1, &lenPrefix, &lenSuffix,'>')) goto done_prefixes; break; case '=': case ':': lenPrefix = j+1; break; default: break; } } done_prefixes: if(lenPrefix != 0 || lenSuffix != 0) { /* TODO: not only print here, but let user override * (in upcoming "interactive" mode) */ printPrefixes(node, lenPrefix, lenSuffix); disjoinCommon(node, lenPrefix, lenSuffix); } } /* if all terminals, squash siblings. It is too dangerous to * do this while creating the tree, but after it has been created * such siblings are really just different values. */ void squashTerminalSiblings(logrec_node_t *const __restrict__ node) { if(!node->sibling) return; int nSiblings = 0; for(logrec_node_t *n = node ; n ; n = n->sibling) { if(n->child || n->nwords > 1) return; nSiblings++; } node->words = realloc(node->words, sizeof(struct wordinfo *) * (node->nwords + nSiblings)); for(logrec_node_t *n = node->sibling ; n ; n = n->sibling) { if(optPrintDebugOutput) printf("add to idx %d: '%s'\n", node->nwords, n->words[0]->word);fflush(stdout); node->words[node->nwords++] = n->words[0]; n->words[0] = NULL; } node->sibling = NULL; // TODO: fix memory leak } /* reprocess tree to check subword creation */ void treeDetectSubwords(logrec_node_t *node) { if(node == NULL) return; reportProgress("subword detection"); squashTerminalSiblings(node); while(node != NULL) { checkSubwords(node); //checkPrefixes(node); treeDetectSubwords(node->child); node = node->sibling; } } /* squash a tree, that is combine nodes that point to nodes * without siblings to a single node. */ void treeSquash(logrec_node_t *node) { if(node == NULL) return; reportProgress("squashing"); squashTerminalSiblings(node); const int hasSibling = node->sibling == NULL ? 0 : 1; while(node != NULL) { if(!hasSibling && node->child != NULL && node->nwords == 1 && node->child->sibling == NULL && node->child->nwords == 1 && node->words[0]->word[0] != '%' /* do not combine syntaxes */ && node->child->words[0]->word[0] != '%') { char *newword; if(asprintf(&newword, "%s %s", node->words[0]->word, node->child->words[0]->word)) {}; /* silence cc warning */ if(optPrintDebugOutput) printf("squashing: %s\n", newword); free(node->words[0]->word); node->words[0]->word = newword; node->nterm = node->child->nterm; /* TODO: do not combine terminals! */ logrec_node_t *toDel = node->child; node->child = node->child->child; logrec_delNode(toDel); continue; /* see if we can squash more */ } //checkPrefixes(node); treeSquash(node->child); node = node->sibling; } } void treePrintIndent(const int level, const char indicator) { printf("%2d%c:", level, indicator); for(int i = 0 ; i < level ; ++i) printf(" "); } void treePrintWordinfo(struct wordinfo *const __restrict__ wi) { printf("%s", wi->word); if(wi->flags.isSubword) printf(" {subword}"); if(wi->occurs > 1) printf(" {%d}", wi->occurs); } void treePrint(logrec_node_t *node, const int level) { if(!optPrintTree) return; reportProgress("print"); while(node != NULL) { if(optSortMultivalues) qsort(node->words, node->nwords, sizeof(struct wordinfo*), qs_compmi); treePrintIndent(level, 'l'); treePrintWordinfo(node->words[0]); if(node->nterm) printf(" [nterm %d]", node->nterm); printf("\n"); for(int i = 1 ; i < node->nwords ; ++i) { treePrintIndent(level, 'v'); treePrintWordinfo(node->words[i]); printf("\n"); } treePrint(node->child, level + 1); node = node->sibling; } } #if 0 void treeToJSON(logrec_node_t *node, json_object *json) { json_object *newobj; int isArray; reportProgress("convert tree to json"); if(node == NULL) return; if(node->sibling == NULL) { isArray = 0; newobj = json_object_new_object(); } else { isArray = 1; newobj = json_object_new_array(); } while(node != NULL) { treePrintWordinfo(node->words[0]); if(node->nterm) printf(" [nterm %d]", node->nterm); printf("\n"); for(int i = 1 ; i < node->nwords ; ++i) { treePrintIndent(level, 'v'); treePrintWordinfo(node->words[i]); printf("\n"); } treePrint(node->child, level + 1); node = node->sibling; } } #endif #if 0 void treePrintTerminalsNonRoot(logrec_node_t *__restrict__ node, const char *const __restrict__ beginOfMsg) { const char *msg = NULL; while(node != NULL) { const char *tail; if(node == root) { msg = ""; } else { if(node->nwords > 1) { tail = "%word%"; } else { tail = node->words[0]->word; } free((void*)msg); asprintf((char**)&msg, "%s%s%s", beginOfMsg, tail, (node->words[0]->flags.isSubword) ? "" : " "); if(node->nterm) printf("%6d times:%s\n", node->nterm, msg); } treePrintTerminalsNonRoot(node->child, msg); node = node->sibling; } } void treePrintTerminals(logrec_node_t *__restrict__ node) { /* we need to strip "[ROOT]" from the node value. Note that it may * have been combined with some other value during tree squash. */ const char *beginOfMsg = node->words[0]->word + 6 /* "[ROOT]"! */; while(node != NULL) { treePrintTerminalsNonRoot(node->child, ""); node = node->sibling; } } #endif void treeCreateRuleTableNonRoot(logrec_node_t *__restrict__ node, rule_table_t *const __restrict__ rt, const char *const __restrict__ beginOfMsg) { const char *msg = NULL; while(node != NULL) { const char *tail; if(node->nwords > 1) { tail = "%MULTIVALUE%"; } else { tail = node->words[0]->word; } free((void*)msg); if(asprintf((char**)&msg, "%s%s%s", beginOfMsg, tail, (node->words[0]->flags.isSubword) ? "" : " ") == -1) {}; /* silence cc warning */ if(node->nterm) { reportProgress("rule table create"); rule_table_etry_t *const rt_etry = ruleTableEtryCreate(rt); rt_etry->ntimes = node->nterm; rt_etry->rule = strdup(msg); } treeCreateRuleTableNonRoot(node->child, rt, msg); node = node->sibling; } } rule_table_t * treeCreateRuleTable(logrec_node_t *__restrict__ node) { /* we need to strip "[ROOT]" from the node value. Note that it may * have been combined with some other value during tree squash. */ const char *beginOfMsg = node->words[0]->word + 6 /* "[ROOT]"! */; rule_table_t *const __restrict__ rt = ruleTableCreate(); while(node != NULL) { treeCreateRuleTableNonRoot(node->child, rt, beginOfMsg); node = node->sibling; } return rt; } /* TODO: move wordlen to struct wordinfo? */ void /* NOTE: bDetectStacked is just a development aid, it permits us to write * a first version which does not detect multi-node items that would * go to the stack and require more elaborate handling. TODO: remove that * restriction. * TODO: check: we may remove stacked mode due to new subword algo (if it stays!) */ wordDetectSyntax(struct wordinfo *const __restrict__ wi, const size_t wordlen, const int bDetectStacked) { size_t nproc; size_t constzero = 0; /* the default lognorm parsers need this */ if(syntax_posint(wi->word, wordlen, NULL, &nproc) && nproc == wordlen) { free(wi->word); wi->word = strdup("%posint%"); wi->flags.isSpecial = 1; goto done; } if(ln_parseTime24hr(wi->word, wordlen, &constzero, NULL, &nproc, NULL) == 0 && nproc == wordlen) { free(wi->word); wi->word = strdup("%time-24hr%"); wi->flags.isSpecial = 1; goto done; } /* duration needs to go after Time24hr, as duration would accept * Time24hr format, whereas duration usually starts with a single * digit and so Tim24hr will not pick it. Still we may get false * detection for durations > 10hrs, but so is it... */ if(ln_parseDuration(wi->word, wordlen, &constzero, NULL, &nproc, NULL) == 0 && nproc == wordlen) { free(wi->word); wi->word = strdup("%duration%"); wi->flags.isSpecial = 1; goto done; } if(syntax_ipv4(wi->word, wordlen, NULL, &nproc)) { if(nproc == wordlen) { free(wi->word); wi->word = strdup("%ipv4%"); wi->flags.isSpecial = 1; goto done; } if(bDetectStacked && wi->word[nproc] == '/') { size_t strtnxt = nproc + 1; if(syntax_posint(wi->word+strtnxt, wordlen-strtnxt, NULL, &nproc)) if(strtnxt+nproc == wordlen) { free(wi->word); wi->word = strdup("%ipv4%"); wi->flags.isSubword = 1; wi->flags.isSpecial = 1; struct wordinfo *wit; wit = wordinfoNew("%posint%"); wit->flags.isSubword = 1; wit->flags.isSpecial = 1; wordstackPush(wit); wit = wordinfoNew("/"); wit->flags.isSubword = 1; wordstackPush(wit); goto done; } } } if(ln_parseKernelTimestamp(wi->word, wordlen, &constzero, NULL, &nproc, NULL) == 0 && nproc == wordlen) { free(wi->word); wi->word = strdup("%kernel-timestamp%"); wi->flags.isSpecial = 1; goto done; } done: return; } struct wordinfo * getWord(char **const line) { struct wordinfo *wi = wordstackPop(); if(wi != NULL) return wi; char *ln = *line; if(*ln == '\0') return NULL; size_t i; for(i = 0 ; ln[i] && isspace(ln[i]) ; ++i) /* EMPTY - skip spaces */; const size_t begin_word = i; for( ; ln[i] && !isspace(ln[i]) ; ++i) { #if 0 /* turn on for subword detection experiment */ if(ln[i] == ':' || ln[i] == '=' || ln[i] == '/' || ln[i] == '[' || ln[i] == ']' || ln[i] == '(' || ln[i] == ')') { wi = wordinfoNew(NULL); wi->word = malloc(2); wi->word[0] = ln[i]; wi->word[1] = '\0'; wordstackPush(wi); // TODO: if we continue with this approach, we must indicate that // this is a subword. /* mimic word delimiter, will be skipped over in next run: */ ln[i] = ' '; break; } #endif } if(begin_word == i) /* only trailing spaces? */ return NULL; const size_t wordlen = i - begin_word; wi = wordinfoNew(NULL); wi->word = malloc(wordlen + 1); memcpy(wi->word, ln+begin_word, wordlen); wi->word[wordlen] = '\0'; if(wi->word[0] == '%') /* assume already token [TODO: improve] */ goto done; wordDetectSyntax(wi, wordlen, 1); done: *line = ln+i; return wi; } logrec_node_t * treeAddToLevel(logrec_node_t *const level, /* current node */ struct wordinfo *const wi, struct wordinfo *const nextwi ) { logrec_node_t *existing, *prev = NULL; for(existing = level->child ; existing != NULL ; existing = existing->sibling) { struct wordinfo *wi_val; if((wi_val = logrec_hasWord(existing, wi->word)) != NULL) { wi_val->occurs++; break; } prev = existing; } if(existing == NULL && nextwi != NULL) { /* we check if the next word is the same, if so, we can * just add this as a different value. */ logrec_node_t *child; for(child = level->child ; child != NULL ; child = child->sibling) { if(child->child != NULL && !strcmp(child->child->words[0]->word, nextwi->word)) break; } if(child != NULL) { logrec_addWord(child, wi); existing = child; } } if(existing == NULL) { existing = logrec_newNode(wi, level); if(prev == NULL) { /* first child of parent node */ level->child = existing; } else { /* potential new sibling */ prev->sibling = existing; } } return existing; } void treeAddLine(char *ln) { struct wordinfo *wi; struct wordinfo *nextwi; /* we need one-word lookahead for structure tree */ logrec_node_t *level = root; nextwi = getWord(&ln); while(1) { wi = nextwi; if(wi == NULL) { ++level->nterm; break; } nextwi = getWord(&ln); level = treeAddToLevel(level, wi, nextwi); } } void preprocessLine(const char *const __restrict__ buf, const size_t buflen, char *const bufout) { static int lnCnt = 1; size_t nproc; char *tocopy; size_t tocopylen; size_t iout; iout = 0; for(size_t i = 0 ; i < buflen ; ) { /* in this stage, we must only detect syntaxes that we are * very sure to correctly detect AND that *spawn multiple * words*. Otherwise, it is safer to detect them on a * word basis. */ if(ln_parseRFC3164Date(buf, buflen, &i, NULL, &nproc, NULL) == 0) { tocopy = "%date-rfc3164%"; } else if(ln_parseRFC5424Date(buf, buflen, &i, NULL, &nproc, NULL) == 0) { tocopy = "%date-rfc5424%"; } else if(ln_parseISODate(buf, buflen, &i, NULL, &nproc, NULL) == 0) { tocopy = "%date-iso%"; } else if(ln_parsev2IPTables(buf, buflen, &i, NULL, &nproc, NULL) == 0) { tocopy = "%v2-iptables%"; } else if(ln_parseNameValue(buf, buflen, &i, NULL, &nproc, NULL) == 0) { tocopy = "%name-value-list%"; } else if(ln_parseCiscoInterfaceSpec(buf, buflen, &i, NULL, &nproc, NULL) == 0) { tocopy = "%cisco-interface-spec%"; } else if(ln_parseCEESyslog(buf, buflen, &i, NULL, &nproc, NULL) == 0) { tocopy = "%cee-syslog%"; } else if(ln_parseJSON(buf, buflen, &i, NULL, &nproc, NULL) == 0) { tocopy = "%json%"; } else { tocopy = NULL; nproc = 1; } /* copy to output buffer */ if(tocopy == NULL) { bufout[iout++] = buf[i]; } else { tocopylen = strlen(tocopy); // do this in lower lever memcpy(bufout+iout, tocopy, tocopylen); iout += tocopylen; } i += nproc; } bufout[iout] = '\0'; ++lnCnt; } int processFile(FILE *fp) { char lnbuf[MAXLINE]; char lnpreproc[MAXLINE]; while(!feof(fp)) { reportProgress("reading"); size_t i; for(i = 0 ; i < sizeof(lnbuf)-1 ; ++i) { const int c = fgetc(fp); if(c == EOF || c == '\n') break; lnbuf[i] = c; } lnbuf[i] = '\0'; if(i > 0) { //processLine(lnbuf, i, &logrec); //logrecPrint(logrec); preprocessLine(lnbuf, i, lnpreproc); treeAddLine(lnpreproc); } } treePrint(root, 0); treeDetectSubwords(root); treeSquash(root); treePrint(root, 0); rule_table_t *const rt = treeCreateRuleTable(root); reportProgress("sorting rule table"); qsort(rt->entries, (size_t) rt->nxtEtry, sizeof(rule_table_etry_t*), qs_comp_rt_etry); ruleTablePrint(rt); ruleTableDestroy(rt); //treePrintTerminals(root); reportProgress(NULL); return 0; } #define OPT_PRINT_TREE 1000 #define OPT_PRINT_DEBUG_OUTPUT 1001 #define OPT_SORT_MULTIVALUES 1002 int main(int argc, char *argv[]) { int r; int ch; static const struct option longopts[] = { { "report-progress", no_argument, 0, 'p' }, { "print-tree", no_argument, 0, OPT_PRINT_TREE }, { "print-debug-output", no_argument, 0, OPT_PRINT_DEBUG_OUTPUT }, { "sort-multivalues", required_argument,0, OPT_SORT_MULTIVALUES }, { NULL, 0, 0, 0 } }; setvbuf(stdout, NULL, _IONBF, 0); while ((ch = getopt_long(argc, argv, "p", longopts, NULL)) != -1) { switch (ch) { case 'p': /* file to log */ displayProgress = 1; break; case OPT_PRINT_TREE: optPrintTree = 1; break; case OPT_PRINT_DEBUG_OUTPUT: optPrintDebugOutput = 1; break; case OPT_SORT_MULTIVALUES: if(!strcmp(optarg, "enabled")) optSortMultivalues = 1; else if(!strcmp(optarg, "disabled")) optSortMultivalues = 0; else { fprintf(stderr, "invalid value '%s' for --sort-multivalues." "Valid: \"enabled\", \"disabled\"\n", optarg); exit(1); } break; case '?': default: // usage(stderr); fprintf(stderr, "invalid option"); break; } } root = logrec_newNode(wordinfoNew(strdup("[ROOT]")), NULL); r = processFile(stdin); return r; } liblognorm-2.0.8/tools/squashml.c000066400000000000000000000026421511425433100170310ustar00rootroot00000000000000/* a small tool to squash multiline message - probably to be removed * later and integrated into the "mainstream" tools. * Copyright (C) 2015 by Rainer Gerhards * Released under ASL 2.0 */ #include #include #include #include #include char * getmsg(regex_t *const preg, char *const buf, size_t len) { static size_t lenln = 0; static char lnbuf[1024*64]; size_t iDst = 0; int nlines = 0; if(lenln) { /* have previous segment? */ memcpy(buf+iDst, lnbuf, lenln); iDst += lenln; ++nlines; } while(fgets(lnbuf, sizeof(lnbuf), stdin)) { lenln = strlen(lnbuf); if(lnbuf[lenln-1] == '\n') { lnbuf[lenln-1] = '\0'; lenln--; } const int is_match = !regexec(preg, lnbuf, 0, NULL, 0); if(is_match) { break; /* previous message complete */ } else { if(iDst != 0) { buf[iDst++] = '\\'; buf[iDst++] = 'n'; } memcpy(buf+iDst, lnbuf, lenln); iDst += lenln; ++nlines; } } if(nlines == 0 && lenln > 0) { /* handle single lines */ memcpy(buf+iDst, lnbuf, lenln); iDst += lenln; lenln = 0; } buf[iDst] = '\0'; } int main(int argc, char *argv[]) { if(argc != 2) { fprintf(stderr, "usage: squashml regex\n"); exit(1); } regex_t preg; if(regcomp(&preg, argv[1], REG_EXTENDED)) { perror("regcomp"); exit(1); } char msg[1024*256]; while(!feof(stdin)) { getmsg(&preg, msg, sizeof(msg)); printf("%s\n", msg); } } liblognorm-2.0.8/tools/syntaxes.c000066400000000000000000000042241511425433100170500ustar00rootroot00000000000000/* Syntax "detectors" * * Copyright 2015 Rainer Gerhards * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "config.h" #include #include #include "syntaxes.h" /* returns -1 if no integer found, else integer */ static int64_t getPosInt(const char *const __restrict__ buf, const size_t buflen, size_t *const __restrict__ nprocessed) { int64_t val = 0; size_t i; for(i = 0 ; i < buflen ; ++i) { if('0' <= buf[i] && buf[i] <= '9') val = val*10 + buf[i]-'0'; else break; } *nprocessed = i; if(i == 0) val = -1; return val; } /* 1 - is IPv4, 0 not */ int syntax_ipv4(const char *const __restrict__ buf, const size_t buflen, const char *extracted, size_t *const __restrict__ nprocessed) { int64_t val; size_t nproc; size_t i; int r = 0; val = getPosInt(buf, buflen, &i); if(val < 1 || val > 255) goto done; if(buf[i] != '.') goto done; i++; val = getPosInt(buf+i, buflen-i, &nproc); if(val < 0 || val > 255) goto done; i += nproc; if(buf[i] != '.') goto done; i++; val = getPosInt(buf+i, buflen-i, &nproc); if(val < 0 || val > 255) goto done; i += nproc; if(buf[i] != '.') goto done; i++; val = getPosInt(buf+i, buflen-i, &nproc); if(val < 0 || val > 255) goto done; i += nproc; //printf("IP Addr[%zd]: '%s'\n", i, buf); *nprocessed = i; r = 1; done: return r; } /* 1 - is positive integer, 0 not */ int syntax_posint(const char *const __restrict__ buf, const size_t buflen, const char *extracted, size_t *const __restrict__ nprocessed) { int64_t val; size_t i; int r = 0; val = getPosInt(buf, buflen, &i); if(val == -1) goto done; *nprocessed = i; r = 1; done: return r; } liblognorm-2.0.8/tools/syntaxes.h000066400000000000000000000016411511425433100170550ustar00rootroot00000000000000/* Syntax "detectors" * * Copyright 2015 Rainer Gerhards * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* 1 - is IPv4, 0 not */ int syntax_ipv4(const char *const __restrict__ buf, const size_t buflen, const char *extracted, size_t *const __restrict__ nprocessed); int syntax_posint(const char *const __restrict__ buf, const size_t buflen, const char *extracted, size_t *const __restrict__ nprocessed);