pax_global_header00006660000000000000000000000064147650437350014530gustar00rootroot0000000000000052 comment=4a93073ed158f430b9ac04b99cefa286d945c2d7 blackboxprotobuf-1.4.2/000077500000000000000000000000001476504373500151025ustar00rootroot00000000000000blackboxprotobuf-1.4.2/.github/000077500000000000000000000000001476504373500164425ustar00rootroot00000000000000blackboxprotobuf-1.4.2/.github/workflows/000077500000000000000000000000001476504373500204775ustar00rootroot00000000000000blackboxprotobuf-1.4.2/.github/workflows/python-publish.yml000066400000000000000000000035211476504373500242100ustar00rootroot00000000000000# This workflow will upload a Python Package to PyPI when a release is created # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries # This workflow uses actions that are not certified by GitHub. # They are provided by a third-party and are governed by # separate terms of service, privacy policy, and support # documentation. name: Upload Python Package on: release: types: [published] permissions: contents: read jobs: release-build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.x" - name: Build release distributions run: | python -m pip install poetry cd lib poetry build - name: Upload distributions uses: actions/upload-artifact@v4 with: name: release-dists path: lib/dist/ pypi-publish: runs-on: ubuntu-latest needs: - release-build permissions: # IMPORTANT: this permission is mandatory for trusted publishing id-token: write # Dedicated environments with protections for publishing are strongly recommended. # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules environment: name: pypi url: https://pypi.org/project/bbpb//${{ github.event.release.name }} steps: - name: Retrieve release distributions uses: actions/download-artifact@v4 with: name: release-dists path: lib/dist/ - name: Publish release distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: lib/dist/ blackboxprotobuf-1.4.2/.gitignore000066400000000000000000000001631476504373500170720ustar00rootroot00000000000000*.class TODO *.swp .vagrant __pycache__ *.pyc .vscode .hypothesis* test_payload Test_pb2.py Test_pb2_grpc.py *.pem blackboxprotobuf-1.4.2/.gitmodules000066400000000000000000000003271476504373500172610ustar00rootroot00000000000000[submodule "burp-deps/six"] path = burp/deps/six url = https://github.com/benjaminp/six [submodule "lib/tests/deps/protobuf"] path = lib/tests/deps/protobuf url = https://github.com/protocolbuffers/protobuf.git blackboxprotobuf-1.4.2/LICENSE000066400000000000000000000020631476504373500161100ustar00rootroot00000000000000MIT License Copyright (c) 2018-2023 NCC Group Plc Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. blackboxprotobuf-1.4.2/README.md000066400000000000000000000040231476504373500163600ustar00rootroot00000000000000# Blackbox Protobuf **Blackbox Protobuf now has an official package on PyPi under the name `bbpb`. The `blackboxprotobuf` package is an older fork** ## Description Blackbox Protobuf is a set of tools for working with encoded Protocol Buffers (protobuf) without the matching protobuf definition. Protobuf is a binary serialization format from Google which can be used as a more efficient alternative to formats like JSON or XML. Developers can define the message format in a `.proto` file and use the protobuf compiler to generate message handlers in their language of choice. The protobuf encoding is binary, and unlike json/xml not human readable or easy to modify by hand. The format also takes advantage of both sides having the message definition and strips out much of the type information. This is good for efficiency, but increases the difficulty analyzing or modifying the network traffic. Blackbox protobuf is designed to allow working with protocol buffers without the message definition. It was originally implemented as a Burp extension for decoding and modifying messages during mobile pentests, but has also been used for reverse engineering and forensics tooling. ## Tools This repository contains several interfaces for working with protocol buffers: - A jython burp extension in [burp/](https://github.com/nccgroup/blackboxprotobuf/tree/master/burp) - A python library that can be used in other applications in [lib/](https://github.com/nccgroup/blackboxprotobuf/tree/master/lib) - A python-based CLI embedded in the [library](https://github.com/nccgroup/blackboxprotobuf/tree/master/lib/CLI.md) - A mitmproxy addon in [mitmproxy](https://github.com/nccgroup/blackboxprotobuf/tree/master/mitmproxy) ## Documentation In addition to the `README.md` for each tool, the following documentation is available: - [Type Definition Guide](docs/TypeDefs.md) - guide for editing typedefs to fix types and improve readability ## Future Tools Some tooling that may be built on top of blackboxprotobuf in the future: - protobuf type discovery tool blackboxprotobuf-1.4.2/Vagrantfile000066400000000000000000000017641476504373500172770ustar00rootroot00000000000000# -*- mode: ruby -*- # vi: set ft=ruby : Vagrant.configure("2") do |config| config.vm.box = "bento/ubuntu-22.04" config.vm.synced_folder "./", "/vagrant", disabled: true config.vm.synced_folder "./", "/home/vagrant/blackboxprotobuf", type: "rsync" #config.vm.synced_folder "./", "/home/vagrant/blackboxprotobuf" config.vm.provision "shell", inline: <<-SHELL echo "export PATH=/home/vagrant/.local/bin/:$PATH" >> /home/vagrant/.bashrc sudo apt-get update sudo apt-get install -y protobuf-compiler jython python3-pip python2 wget https://bootstrap.pypa.io/pip/2.7/get-pip.py -O /tmp/get-pip2.py python2 /tmp/get-pip2.py python3 -m pip install poetry python2 -m pip install virtualenv SHELL config.vm.provision "shell", privileged: false, inline: <<-SHELL cd blackboxprotobuf/lib #poetry env use python2 && poetry install --no-root poetry env use python3 && poetry install --no-root poetry env use python3 && poetry run mypy --install-types SHELL end blackboxprotobuf-1.4.2/burp/000077500000000000000000000000001476504373500160525ustar00rootroot00000000000000blackboxprotobuf-1.4.2/burp/README.md000066400000000000000000000176261476504373500173450ustar00rootroot00000000000000# BlackBox Protobuf Burp Extension ## Description This is an extension for the intercepting proxy Burp Suite () that allows encoding and decoding arbitrary protocol buffer () messages which may be contained in an intercepted request. It is designed to work without a protobuf definition file (.proto) which may not be available or may be unusable with existing Burp extensions. Background information on Protobuf decoding and a breakdown of the type system and possible type corner cases may be found in the library documentation under: # Usage ## Installation 1. If Burp Suite is not already installed, download it from . 2. Download/Install Jython 2.7+ and configure Burp with the location. See . 3. Clone this repository and then run `git submodule update --init` to install dependencies. 4. Within Burp, navigate to Extender -> Extensions and select "Add". 5. Set "Extension Type" to Python and select the `extender.py` file in the git repository. 6. Click Next and the extension should load. 7. **Note:** [gRPC](https://grpc.io/about/) is supported, but you'll have to enable Burp's HTTP/2 support (under Project Options->HTTP). Also, currently only uncompressed gRPC payloads are supported. If the first byte of the payload is not `0x00`, it's compressed and you'll have to modify the en/decoding code to account for that. ## Editing Messages A new tab will be added to every message window with a content type of "x-protobuf" or "application/protobuf" (this is configurable via `user_funcs.py`). The protobuf message will be parsed to a JSON dictionary with numbered fields as the key. The values can be modified as long as the new value is of the same type. The upper list shows a list of named type definitions that can decode this message. Selecting one will re-decode the message with the new type. Blackboxprotobuf will try to remember the last message chosen for that endpoint. The `new` button will save the current type definition under a new name. The "Validate" button verifies that a modified JSON message can be re-encoded. It's best to use this to validate a message before switching to a different view or sending the message. If you switch away from the tab with an invalid payload, it will raise an error and reset to the original value. The "Edit Type" brings up a window for editing the current message's type definition in a JSON format. Allowing you to change types or name fields. The current message will be decoded with the new type upon saving. If you edit the default type definition for a message, you should use the `New` button to save it, or it will be forgotten on the next message. The "Reset Message" button will revert the protobuf message to the original decoded value. The `Clear Type` button will reset to a new anonymous type definition. ## Editing Types The type definition for a message can be modified to make protobuf messages easier to work with. This allows you to change how a message is decoded (eg. decode a field as an `sint` instead of the default `int`) and allows you to assigned names to fields to improve readability. Field numbers should not be modified and types should only be changed to types within the same wire type. A full list of wiretypes and sub-types can be found below. The `example_value_ignored` field in the type definition should contain a value from the message to make it easier to locate the right field to modify, but the value itself is ignored when the type definition is parsed. ### Type Reference * Varint - Variable length integers (up to 8 bytes) - `uint` - unsigned, represents positive numbers efficiently, can't represent negative numbers - `int` - (default) signed, but represents negative numbers inefficiently - `sint` - Zig-zag encoding to map unsigned space to signed * Fixed32 - Always 32 bits - `fixed32` - (default) unsigned integer - `sfixed32` - signed integer - `float` - floating point number * Fixed64 - Always 64 bits - `fixed64` - (default) unsigned integer - `sfixed64` - signed integer - `double` - floating point number * Length Delimited - Prefixed by length representing varint - `bytes` - (default) Plain data, used for strings as well - `message` - (detected) Protobuf message. Can contain a nested type definition ('`message_typedef`') or labeled type name ('`message_type_name`') - `string` - Similar to bytes, but will return a string python type - `bytes_hex` - Output binary data as a string of hex characters rather than an escaped string - `packed_*` - Repeated fields of the same type packed into a buffer. Can be combined with any Varint, or fixed wiretype (eg. `packed_fixed32`) * Group (Start/End) - `group` - Deprecated way to group fields. Replaced with nested Protobuf Messages. Not supported ## Protobuf Type Editor Tab Any message definitions saved with a name will be shown in the global "Protobuf Type Editor Tab". This tab allows type definitions to be created, renamed, edited and removed without an active request/response. The "Save All Types"/"Load All Types" buttons can be used to export or import the type definitions as JSON files. This can ensure the types are safely backed up or to share them between instances. Named types should persist in the extension settings between Burp reboots, but if a lot of effort has been put into customizing definitions, it may be a good idea to back them up regularly. Finally, the extension will try to import/export `.proto` files. The `.proto` export will try save all known type definitions into the protobuf type definition format. You should then be able to import the `.proto` files into other tools that expect the original type definitions. The import functionality will attempt to read a `.proto` file and create a Blackbox protobuf type definition from it. This does not support "import" statements and any files referenced by the import statement should be imported first. Both import and export functionality is pretty hacky and may not work for all message types. ## User Functions Some of the behavior of the extension can be changed through the `burp/blackboxprotobuf/burp/user_funcs.py` file. Each function is called by the extension to provide alternative ways to handle a message: * `detect_protobuf` - Customizes how the extension determines if a request/response is a protobuf message. By default, the extension checks for a few content-type headers to know when to parse a request/response as protobuf. This function can be used to check for other headers, parameters or just return True for all messages. Should return `True` if it is protobuf, `False` if it isn't, or `None` to fall back to the content-type check. * `get_protobuf_data` - Customizes the process of retrieving the data from the message. By default the extension will retrieve the binary data from the message body. This function can be used to get data from other location such as a header or parameter. This can also be used to parse non-default encodings. Should return the protobuf data. * `set_protobuf_data` - Customizes how protobuf data is stored back in the request/response once it is re-encoded. Should mirror `get_protobuf_data` and is only necessary if `get_protobuf_data` is customized. * `hash_message` - Customizes how the extension identifies which message type to use for a request/response. By default, the extension uses a combination of the path and whether it is a request or a response. If the application has a better indicator, such as a `MessageType` header or parameter, then this function can return that as a key. The returned value is just used as a key to a dict/hashmap so can be any arbitrary value, but should be a string value so that it can be serialized as JSON for persistence. blackboxprotobuf-1.4.2/burp/blackboxprotobuf/000077500000000000000000000000001476504373500214205ustar00rootroot00000000000000blackboxprotobuf-1.4.2/burp/blackboxprotobuf/__init__.py000066400000000000000000000040311476504373500235270ustar00rootroot00000000000000# Copyright (c) 2018-2023 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import sys import inspect # Add correct directory to sys.path _BASE_DIR = os.path.abspath( os.path.dirname(inspect.getfile(inspect.currentframe())) + "../../../" ) sys.path.insert(0, _BASE_DIR + "/lib/") sys.path.insert(0, _BASE_DIR + "/burp/deps/six/") sys.path.insert(0, _BASE_DIR + "/burp/deps/protobuf/python/") # extend_path looks for other 'blackboxprotobuf' modules in the sys.path and # adds them to __path__ from pkgutil import extend_path __path__ = extend_path(__path__, __name__) # Hack to fix loading protobuf libraries within Jython. See https://github.com/protocolbuffers/protobuf/issues/7776 def fix_protobuf(): import six u = six.u def new_u(s): if s == r"[\ud800-\udfff]": # Don't match anything return "$^" else: return u(s) six.u = new_u fix_protobuf() # mirror what we do in lib so we can use blackboxprotobuf. from blackboxprotobuf.lib.api import * blackboxprotobuf-1.4.2/burp/blackboxprotobuf/burp/000077500000000000000000000000001476504373500223705ustar00rootroot00000000000000blackboxprotobuf-1.4.2/burp/blackboxprotobuf/burp/__init__.py000066400000000000000000000021111476504373500244740ustar00rootroot00000000000000# Copyright (c) 2018-2023 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. blackboxprotobuf-1.4.2/burp/blackboxprotobuf/burp/editor.py000066400000000000000000001041021476504373500242260ustar00rootroot00000000000000"""Module contains classes required to create Protobuf editor tabs.""" # Copyright (c) 2018-2023 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import re import six import traceback import base64 import zlib import burp import copy import struct import blackboxprotobuf from javax.swing import JSplitPane, JPanel, JButton, BoxLayout, JOptionPane from javax.swing import ( Box, JTextField, JScrollPane, JList, ListSelectionModel, ListModel, ) from javax.swing.event import ListSelectionListener, ListDataEvent, ListDataListener from java.awt import Component, Dimension, FlowLayout from java.awt.event import ActionListener from javax.swing.border import EmptyBorder from blackboxprotobuf.burp import user_funcs from blackboxprotobuf.burp import typedef_editor from blackboxprotobuf.lib import payloads from blackboxprotobuf.lib.config import default as default_config from blackboxprotobuf.lib.exceptions import ( BlackboxProtobufException, DecoderException, EncoderException, ) NAME_REGEX = re.compile(r"\A[a-zA-Z_][a-zA-Z0-9_]*\Z") class ProtoBufEditorTabFactory(burp.IMessageEditorTabFactory): """Just returns instances of ProtoBufEditorTab""" def __init__(self, extender, callbacks): self._callbacks = callbacks self._extender = extender def createNewInstance(self, controller, editable): """Return new instance of editor tab for a new message""" return ProtoBufEditorTab(self._extender, controller, editable, self._callbacks) class ProtoBufEditorTab(burp.IMessageEditorTab): """Tab in interceptor/repeater for editing protobuf message. Decodes the message to JSON and back for editing. The message type definition is attached to this object for re-encoding or editing. """ def __init__(self, extension, controller, editable, callbacks): self._callbacks = callbacks self._extension = extension self._callbacks = extension.callbacks self._helpers = extension.helpers self._controller = controller self._text_editor = self._callbacks.createTextEditor() self._text_editor.setEditable(editable) self._editable = editable self._last_valid_type_index = None self._filtered_message_model = FilteredMessageModel( extension.known_message_model, self._callbacks ) self._type_list_component = JList(self._filtered_message_model) self._type_list_component.setSelectionMode(ListSelectionModel.SINGLE_SELECTION) self._type_list_component.addListSelectionListener(TypeListListener(self)) self._new_type_field = JTextField() self._component = JSplitPane(JSplitPane.HORIZONTAL_SPLIT) self._component.setLeftComponent(self._text_editor.getComponent()) self._component.setRightComponent(self.createButtonPane()) self._component.setResizeWeight(0.95) self._message_info = None self._payload_info = None self._last_good_msg = None # (msg, typedef, source) self._decode_task = None def getTabCaption(self): """Return message tab caption""" return "Protobuf" def getMessage(self): """Transform the JSON format back to the binary protobuf message""" # Noticing this gets called twice for some reason # If we haven't finished decoding the message, cancel and return the original content if self._decode_task and ( not self._decode_task.isDone() or self._decode_task.isCancelled() ): self._decode_task.cancel(True) self._decode_task = None # self._callbacks.printOutput( # "Called getMessage before decode task was done, returning original content" # ) return self._message_info.content() if self._last_good is None or not self.isModified(): return self._message_info.content() success = False try: json_data = self._text_editor.getText().tostring() protobuf_data = blackboxprotobuf.protobuf_from_json( json_data, self._last_good.typedef ) success = True self._payload_info.protobuf_data = protobuf_data return self._payload_info.generate_http(self._message_info, self._helpers) except Exception as exc: self._callbacks.printError(traceback.format_exc()) if not success: try: protobuf_data = blackboxprotobuf.protobuf_from_json( self._last_good.message, self._last_good.typedef ) # Put the error here so that we only have one error to the user if the above encoding doesn't work JOptionPane.showMessageDialog( self._component, "Error encoding protobuf as-is. Reset data to previous good state: " + str(exc), ) success = True # Reset the message and protobuf data both self._text_editor.setText(self._last_good.message) self._payload_info.protobuf_data = protobuf_data return self._payload_info.generate_http( self._message_info, self._helpers ) except Exception as exc: self._callbacks.printError(traceback.format_exc()) JOptionPane.showMessageDialog( self._component, "Error encoding protobuf. Setting data to the original message. Error: " + str(exc), ) self._text_editor.setText(self._message_info.content()) return self._message_info.content() def _handle_protobuf( self, message_info, protobuf_data, message_type_in, typedef_source ): """ Sets the protobuf message for the editor. """ try: json_data, message_type = blackboxprotobuf.protobuf_to_json( protobuf_data, message_type_in ) self._last_good = LastGoodData(json_data, message_type, typedef_source) self._filtered_message_model.set_new_data(protobuf_data) self._text_editor.setText(json_data) # UI access success = True except Exception as exc: success = False self._callbacks.printError( "Got error decoding protobuf binary: " + traceback.format_exc() ) # Bring out of exception handler to avoid nesting handlers if success: if typedef_source is not None: self.forceSelectType(typedef_source) else: self._type_list_component.clearSelection() elif len(message_type_in) == 0: self._callbacks.printError( "Error decoding protobuf with saved type, trying with empty type" ) self._handle_protobuf(message_info, protobuf_data, {}, None) else: self._callbacks.printError("Error decoding protobuf with empty type") self._text_editor.setText("Error decoding protobuf") def setMessage(self, content, is_request): """ Get the data from the request/response and parse into JSON. """ # Run in a separate thread to avoid hanging the Burp UI # It has been observed that the Burp UI can hang when the message is large # and the decoding process takes a long time message_info = MessageInfo(content, is_request, self._helpers, self._controller) payload_info = PayloadInfo(message_info, self._helpers) message_type, typedef_source = self._get_saved_typedef(message_info) if ( self._decode_task and not self._decode_task.isCancelled() and not self._decode_task.isDone() ): # If we're processing the same message as the running task # check message has # check protobuf data # TODO do we want to check typedef? Can switch away and back to restart it if we need to if ( message_info.message_hash == self._message_info.message_hash and payload_info.raw_data == self._payload_info.raw_data ): # self._callbacks.printOutput( # "Switched to tab that is still running with the same hash and payload. Not cancelling." # ) return else: # self._callbacks.printOutput( # "Have existing task that is still running, cancelling" # ) # Cancel the old task self._decode_task.cancel(True) self._message_info = message_info self._payload_info = payload_info self._last_good_msg = None self._decode_task = None self._text_editor.setText("Please wait...") def run(): try: decoders = payloads.find_decoders(payload_info.raw_data) for decoder in decoders: try: protobuf_data, encoding_alg = decoder(payload_info.raw_data) except BlackboxProtobufException: continue try: self._handle_protobuf( message_info, protobuf_data, message_type, typedef_source, ) # Payload successfully decoded, so we probably have the right wrapper for the payload self._payload_info.protobuf_data = protobuf_data self._payload_info.encoding_alg = encoding_alg return except BlackboxProtobufException: if encoding_alg == "none": # Reraise the exception to the parent context and halt decoding six.reraise(*sys.exc_info()) continue except Exception as ex: # Catch all, otherwise it disappears self._text_editor.setText("Error decoding protobuf") self._callbacks.printError("Error decoding protobuf: %s" % ex) self._decode_task = self._extension.thread_executor.submit(run) def getSelectedData(self): """Get text currently selected in message""" return self._text_editor.getSelectedText() def getUiComponent(self): """Return Java AWT component for this tab""" return self._component def isEnabled(self, content, is_request): """Try to detect a protobuf in the message to enable the tab. Defaults to content-type header of 'x-protobuf'. User overridable in `user_funcs.py` """ # TODO implement some more default checks if is_request: info = self._helpers.analyzeRequest(content) else: info = self._helpers.analyzeResponse(content) if "detect_protobuf" in dir(user_funcs): result = user_funcs.detect_protobuf( content, is_request, info, self._helpers ) if result is not None: return result # Bail early if there is no body if info.getBodyOffset() == len(content): return False protobuf_content_types = [ "protobuf", "grpc", ] # Check all headers for x-protobuf for header in info.getHeaders(): if "content-type" in header.lower(): for protobuf_content_type in protobuf_content_types: if protobuf_content_type in header.lower(): return True return False def isModified(self): """Return if the message was modified""" return self._text_editor.isTextModified() def createButtonPane(self): """Create a new button pane for the message editor tab""" self._button_listener = EditorButtonListener(self) panel = JPanel() panel.setLayout(BoxLayout(panel, BoxLayout.Y_AXIS)) panel.setBorder(EmptyBorder(5, 5, 5, 5)) panel.add(Box.createRigidArea(Dimension(0, 5))) type_scroll_pane = JScrollPane(self._type_list_component) type_scroll_pane.setMaximumSize(Dimension(200, 100)) type_scroll_pane.setMinimumSize(Dimension(150, 100)) panel.add(type_scroll_pane) panel.add(Box.createRigidArea(Dimension(0, 3))) new_type_panel = JPanel() new_type_panel.setLayout(BoxLayout(new_type_panel, BoxLayout.X_AXIS)) new_type_panel.add(self._new_type_field) new_type_panel.add(Box.createRigidArea(Dimension(3, 0))) new_type_panel.add( self.createButton( "New", "new-type", "Save this message's type under a new name" ) ) new_type_panel.setMaximumSize(Dimension(200, 20)) new_type_panel.setMinimumSize(Dimension(150, 20)) panel.add(new_type_panel) button_panel = JPanel() button_panel.setLayout(FlowLayout()) if self._editable: button_panel.add( self.createButton( "Validate", "validate", "Validate the message can be encoded." ) ) button_panel.add( self.createButton("Edit Type", "edit-type", "Edit the message type") ) button_panel.add( self.createButton( "Reset Message", "reset", "Reset the message and undo changes" ) ) button_panel.add( self.createButton( "Clear Type", "clear-type", "Reparse the message with an empty type" ) ) button_panel.setMinimumSize(Dimension(100, 200)) button_panel.setPreferredSize(Dimension(200, 1000)) panel.add(button_panel) return panel def createButton(self, text, command, tooltip): """Create a new button with the given text and command""" button = JButton(text) button.setAlignmentX(Component.CENTER_ALIGNMENT) button.setActionCommand(command) button.addActionListener(self._button_listener) button.setToolTipText(tooltip) return button def validateMessage(self): """Callback for validate button. Attempts to encode the message with the current type definition """ try: json_data = self._text_editor.getText().tostring() typedef = self._last_good.typedef protobuf_data = blackboxprotobuf.protobuf_from_json(json_data, typedef) # If it works, save the message # Don't need to save typeddef because we're using the one from lastgood self._last_good.message = json_data self._payload_info.protobuf_data = protobuf_data except Exception as exc: JOptionPane.showMessageDialog( self._component, "Got exception while trying to encode the message: " + str(exc), ) self._callbacks.printError(traceback.format_exc()) def resetMessage(self): """Drop any changes and revert to the last good message. Callback for "reset" button """ self._text_editor.setText(self._last_good.message) def forceSelectType(self, typename): index = self._filtered_message_model.get_type_index(typename) if index is not None: self._last_valid_type_index = index self._type_list_component.setSelectedIndex(index) def updateTypeSelection(self): """Apply a new typedef based on the selected type in the type list""" # TODO It sucks that we lose the anonymous type if we accidentally # click a type. Maybe we should have an entry in the cached type in the # list? # Or have a warning before switching # Or require a click + a button press? # Check if something is selected if self._type_list_component.isSelectionEmpty(): self._last_valid_type_index = None self._extension.saved_types.pop(self._message_info.message_hash, None) return # TODO won't actually work right if we delete the type we're using a # new type is now in the index if self._last_valid_type_index == self._type_list_component.getSelectedIndex(): # hasn't actually changed since last time we tried # otherwise can trigger a second time when we call setSelectedIndex below on failure return type_name = self._type_list_component.getSelectedValue() # try to catch none here... if not type_name or type_name not in default_config.known_types: return try: self.applyType(default_config.known_types[type_name], type_name) except BlackboxProtobufException as exc: self._callbacks.printError(traceback.format_exc()) if isinstance(exc, EncoderException): JOptionPane.showMessageDialog( self._component, "Error encoding protobuf with previous type: %s" % (exc), ) elif isinstance(exc, DecoderException): JOptionPane.showMessageDialog( self._component, "Error encoding protobuf with type %s: %s" % (type_name, exc), ) # decoder exception means it doesn't match the message that was sucessfully encoded by the prev type self._filtered_message_model.remove_type(type_name) if self._last_valid_type_index is not None: type_name = self._type_list_component.setSelectedIndex( self._last_valid_type_index ) else: self._type_list_component.clearSelection() return self._extension.saved_types[self._message_info.message_hash] = type_name self._last_valid_type_index = self._type_list_component.getSelectedIndex() def editType(self, typedef, source): """Apply and save the new typedef""" # Try to apply the type first try: # TODO Background this like with handle_protobuf? I think we need # to be more confident that we can validate the typedef without # decoding first # The decoding here throws an exception that will prevent us from # closing the typedef editor window if it's not valid self.applyType(typedef, source) except BlackboxProtobufException as exc: self._callbacks.printError("Got exception trying to apply edited typedef.") JOptionPane.showMessageDialog( self._component, "Error decoding the protobuf with the new type: %s" % (exc), ) return if source is None: # Anonymous typedef tied to message hash # save the typedef self._extension.saved_types[self._message_info.message_hash] = typedef else: # Named typedef # save under known typedefs and save the name in source default_config.known_types[source] = typedef self._extension.saved_types[self._message_info.message_hash] = source def applyType(self, typedef, source): """Apply a new typedef to the message. Throws an exception if type is invalid.""" # Convert to protobuf as old type and re-interpret as new type old_typedef = self._last_good.typedef json_data = self._text_editor.getText().tostring() protobuf_data = blackboxprotobuf.protobuf_from_json(json_data, old_typedef) self._payload_info.protobuf_data = protobuf_data new_json, new_typedef = blackboxprotobuf.protobuf_to_json( protobuf_data, typedef ) self._last_good = LastGoodData(new_json, new_typedef, source) self._filtered_message_model.set_new_data(protobuf_data) self._text_editor.setText(str(new_json)) # We do not try to remember the type here, this should be handled by the caller def saveAsNewType(self): """Copy the current type into known_messages""" name = self._new_type_field.getText().strip() if not NAME_REGEX.match(name): JOptionPane.showMessageDialog( self._component, "%s is not a valid " "message name. Message names should be alphanumeric." % name, ) return if name in default_config.known_types: JOptionPane.showMessageDialog( self._component, "Message name %s is " "already taken." % name ) return typedef = self._last_good.typedef # Do a deep copy on the dictionary so we don't accidentally modify others default_config.known_types[name] = copy.deepcopy(typedef) self._last_good.source = name # remember the source, typedef is still the same # update the list of messages. This should trickle down to known message model self._extension.known_message_model.addElement(name) self._new_type_field.setText("") self._extension.saved_types[self._message_info.message_hash] = name # force select our new type self.forceSelectType(name) def clearType(self): self.applyType({}, None) self._type_list_component.clearSelection() self._new_type_field.setText("") self._extension.saved_types.pop(self._message_info.message_hash, None) def open_typedef_window(self): typedef = self._last_good.typedef source = self._last_good.source self._extension.open_typedef_editor(typedef, source, self.editType) def _get_saved_typedef(self, message_info): # Get the typedef we have saved for this message # It can be anonymous, but saved based on the message hash # Or it can be named # Return typedef, typename tuple if message_info.message_hash in self._extension.saved_types: saved_type = self._extension.saved_types[message_info.message_hash] if isinstance(saved_type, dict): return saved_type, None elif saved_type in default_config.known_types: typename = saved_type typedef = default_config.known_types[typename] return typedef, typename else: # We had a type, but it isn't a dict and not in known types # Error, so clear self._extension.saved_types.pop(message_info.message_hash, None) self._callbacks.printError( "Found unknown saved type: %s for %s" % (saved_type, message_info.message_hash) ) else: return {}, None class EditorButtonListener(ActionListener): """Callback listener for buttons in the message editor tab""" def __init__(self, editor_tab): self._editor_tab = editor_tab def actionPerformed(self, event): """Called when when a button in the message editor is pressed""" if event.getActionCommand() == "validate": self._editor_tab.validateMessage() elif event.getActionCommand() == "reset": self._editor_tab.resetMessage() elif event.getActionCommand() == "edit-type": self._editor_tab.open_typedef_window() elif event.getActionCommand() == "new-type": self._editor_tab.saveAsNewType() elif event.getActionCommand() == "clear-type": self._editor_tab.clearType() class TypeListListener(ListSelectionListener): """Callback listener for when a new type is selected form the list""" def __init__(self, editor_tab): self._editor_tab = editor_tab def valueChanged(self, event): if event.getValueIsAdjusting(): return self._editor_tab.updateTypeSelection() class FilteredMessageModel(ListModel, ListDataListener): """Listens to a java ListModel and keeps a subset with just valid types for a message. """ def __init__(self, parent, callbacks): self._callbacks = callbacks self._data = None self._parent_model = parent self._listeners = [] # list of types in the parent model self._types = [] self._parent_types = set(parent.elements()) self._rejected_types = set() self._working_types = set() self._parent_model.addListDataListener(self) def set_new_data(self, data): self._data = data # clear caches self._working_types.clear() self._rejected_types.clear() # recheck all the types with the new data for typename in self._types[:]: if not self._check_type(typename): removed_index = self._types.index(typename) self._types.remove(typename) event = ListDataEvent( self, ListDataEvent.INTERVAL_REMOVED, removed_index, removed_index ) self._send_event(event) interval_start = len(self._types) for typename in self._parent_types: if typename not in self._types and self._check_type(typename): self._types.append(typename) if len(self._types) > interval_start: event = ListDataEvent( self, ListDataEvent.INTERVAL_ADDED, interval_start, len(self._types) - 1 ) self._send_event(event) def get_type_index(self, typename): if typename in self._types: return self._types.index(typename) return None def remove_type(self, typename): # if we failed to apply a type, then this lets us remove it if typename not in self._types: return type_index = self._types.index(typename) # delete by index because we have it del self._types[type_index] self._working_types.remove(typename) self._rejected_types.add(typename) event = ListDataEvent( self, ListDataEvent.INTERVAL_REMOVED, type_index, type_index ) self._send_event(event) def update_types(self): new_parent_types = set(self._parent_model.elements()) added_parent_types = new_parent_types - self._parent_types removed_parent_types = self._parent_types - new_parent_types self._parent_types = new_parent_types for type_name in removed_parent_types: if type_name in self._types: removed_index = self._types.index(type_name) self._types.remove(type_name) event = ListDataEvent( self, ListDataEvent.INTERVAL_REMOVED, removed_index, removed_index ) self._send_event(event) interval_start = len(self._types) for type_name in added_parent_types: if type_name not in self._types and self._check_type(type_name): self._types.append(type_name) # not sure how much effort we want to put into the events. could just mark everything as changed all the time if len(self._types) > interval_start: # if we didn't remove anything, then just issue an added event? event = ListDataEvent( self, ListDataEvent.INTERVAL_ADDED, interval_start, len(self._types) - 1 ) self._send_event(event) def _send_event(self, event): event_type = event.getType() if event_type == ListDataEvent.CONTENTS_CHANGED: for listener in self._listeners: listener.contentsChanged(event) elif event_type == ListDataEvent.INTERVAL_ADDED: for listener in self._listeners: listener.intervalAdded(event) elif event_type == ListDataEvent.INTERVAL_REMOVED: for listener in self._listeners: listener.intervalRemoved(event) def _check_type(self, typename): # TODO this hangs the UI as well # TODO would be better to check by comparing typedefs instead of trying # to decode if typename in self._rejected_types: return False if typename in self._working_types: return True # if we don't have data yet, just quit early if not self._data: return False if typename not in default_config.known_types: return False typedef = default_config.known_types[typename] try: _, _ = blackboxprotobuf.protobuf_to_json(self._data, typedef) except BlackboxProtobufException as exc: self._callbacks.printError(traceback.format_exc()) self._rejected_types.add(typename) return False self._working_types.add(typename) return True def addListDataListener(self, listener): self._listeners.append(listener) def getElementAt(self, i): return self._types[i] def getSize(self): return len(self._types) def removeListDataListener(self, listener): self._listeners.remove(listener) # data listener stuff def contentsChanged(self, event): self.update_types() def intervalAdded(self, event): self.update_types() def intervalRemoved(self, event): self.update_types() class MessageInfo: """This class parses the data we get from burp for us to use throughout the processing""" def __init__(self, content, is_request, helpers, controller): self.is_request = is_request if is_request: self.request = content else: self.request = controller.getRequest() self.response = content self.response_content_info = helpers.analyzeResponse(content) self.request_content_info = helpers.analyzeRequest( controller.getHttpService(), self.request ) self.message_hash = self._message_hash(helpers) def content(self): if self.is_request: return self.request else: return self.response def content_info(self): if self.is_request: return self.request_content_info else: return self.response_content_info def _message_hash(self, helpers): """Compute an "identifier" for the message which is used for sticky type definitions. User modifiable """ message_hash = None if "hash_message" in dir(user_funcs): message_hash = user_funcs.hash_message( self.content(), self.is_request, self.content_info(), helpers, self.request, self.request_content_info, ) if message_hash is None: # Base it off just the URL and request/response url = self.request_content_info.getUrl() message_hash = ":".join( [url.getAuthority(), url.getPath(), str(self.is_request)] ) return message_hash class PayloadInfo: """This class stores the latest payload data and functions to transform it to or from a HTTP message""" def __init__(self, message_info, helpers): self.raw_data = None # Raw data from payload # These attributes must be set from outside this payload self.encoding_alg = None self.protobuf_data = None # last known good encoded protobuf payload self.parse_http(message_info, helpers) def parse_http(self, message_info, helpers): raw_data = None if "get_protobuf_data" in dir(user_funcs): raw_data = user_funcs.get_protobuf_data( message_info.content(), message_info.is_request, message_info.content_info(), helpers, message_info.request, message_info.request_content_info, ) if raw_data is None: raw_data = message_info.content()[ message_info.content_info().getBodyOffset() : ].tostring() self.raw_data = raw_data def generate_http(self, message_info, helpers): if "set_protobuf_data" in dir(user_funcs): result = user_funcs.set_protobuf_data( self.protobuf_data, message_info.content(), message_info.is_request, message_info.content_info(), helpers, message_info.request, message_info.request_content_info, ) if result is not None: return result if self.protobuf_data is None: raise BlackboxProtobufException( "Error generating HTTP body. PayloadInfo does not have valid protobuf data to encode" ) raw_data = payloads.encode_payload(self.protobuf_data, self.encoding_alg) headers = message_info.content_info().getHeaders() return helpers.buildHttpMessage(headers, str(raw_data)) class LastGoodData: """This class stores data about the last now valid combination of message and typedef""" def __init__(self, message, typedef, source): self.message = message self.typedef = typedef self.source = source blackboxprotobuf-1.4.2/burp/blackboxprotobuf/burp/extender.py000066400000000000000000000112061476504373500245600ustar00rootroot00000000000000"""Adds the top level type definition editor and registers the protobuf message editor factory for individual tabs """ # Copyright (c) 2018-2023 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import inspect import os import sys import traceback import burp import json from javax.swing import DefaultListModel from java.util.concurrent import Executors # Add correct directory to sys.path _BASE_DIR = os.path.abspath( os.path.dirname(inspect.getfile(inspect.currentframe())) + "../../../../" ) sys.path.insert(0, _BASE_DIR + "/burp/") import blackboxprotobuf from blackboxprotobuf.lib.config import default as default_config from blackboxprotobuf.burp import editor, typedef_tab, typedef_editor EXTENSION_NAME = "BlackboxProtobuf" class BurpExtender(burp.IBurpExtender, burp.IExtensionStateListener): """Primary extension class. Sets up all other functionality.""" def __init__(self): self.open_windows = [] self.callbacks = None self.helpers = None self.saved_types = {} self.suite_tab = None # global list of known messages for all views # this should mirror what is in blackboxprotobuf.known_messages # TODO bundle them together so it doesn't have to be manually updated self.known_message_model = DefaultListModel() self.refresh_message_model() self.thread_executor = Executors.newCachedThreadPool() def refresh_message_model(self): self.known_message_model.clear() for name in default_config.known_types.keys(): self.known_message_model.addElement(name) def registerExtenderCallbacks(self, callbacks): """Called by burp. Collects callback object and sets up UI""" try: callbacks.registerExtensionStateListener(self) self.callbacks = callbacks self.helpers = callbacks.getHelpers() callbacks.setExtensionName(EXTENSION_NAME) callbacks.registerMessageEditorTabFactory( editor.ProtoBufEditorTabFactory(self, callbacks) ) self.suite_tab = typedef_tab.TypeDefinitionTab(self, callbacks) callbacks.addSuiteTab(self.suite_tab) self.loadKnownMessages() self.refresh_message_model() except Exception as exc: self.callbacks.printError(traceback.format_exc()) raise exc def loadKnownMessages(self): message_json = self.callbacks.loadExtensionSetting("known_types") if message_json: default_config.known_types.update(json.loads(message_json)) saved_types = self.callbacks.loadExtensionSetting("saved_type_map") if saved_types: self.saved_types.update(json.loads(saved_types)) def saveKnownMessages(self): # TODO might be good to cll this more often (eg. when messages are updated) # save the known messages self.callbacks.saveExtensionSetting( "known_types", json.dumps(default_config.known_types) ) self.callbacks.saveExtensionSetting( "saved_type_map", json.dumps(self.saved_types) ) def extensionUnloaded(self): self.thread_executor.shutdownNow() self.saveKnownMessages() for window in self.open_windows: window.exitTypeWindow() def open_typedef_editor(self, message_type, source, callback): self.open_windows = [window for window in self.open_windows if window.is_open] window = typedef_editor.TypeEditorWindow( self.callbacks, message_type, source, callback, ) window.show() self.open_windows.append(window) blackboxprotobuf-1.4.2/burp/blackboxprotobuf/burp/typedef_editor.py000066400000000000000000000147541476504373500257630ustar00rootroot00000000000000"""Contains classes for a window for editing type definitions. Called from both the message editor tab and from the suite tabs. """ # Copyright (c) 2018-2023 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import traceback import json import blackboxprotobuf from java.awt import Component, Dimension, Frame from java.awt.event import ActionListener, WindowEvent from javax.swing import ( JSplitPane, JPanel, JButton, BoxLayout, JOptionPane, JDialog, Box, JTextField, ) class TypeEditorWindow(JDialog): """New free-standing window for editing a specified type definition. Will callback into the calling class when the type is saved """ def __init__(self, burp_callbacks, typedef, source, callback): burp_window = None for frame in Frame.getFrames(): if "Burp Suite" in frame.getName(): burp_window = frame break JDialog.__init__(self, burp_window) self._burp_callbacks = burp_callbacks self._type_callback = callback self.setSize(1000, 700) self._original_typedef = typedef self._type_source = source self._original_json = json.dumps(self._original_typedef, indent=4) self._type_editor = burp_callbacks.createTextEditor() self._type_editor.setEditable(True) self._type_editor.setText(self._original_json) splitPane = JSplitPane(JSplitPane.HORIZONTAL_SPLIT) splitPane.setLeftComponent(self._type_editor.getComponent()) splitPane.setRightComponent(self.createButtonPane()) splitPane.setResizeWeight(0.8) self.add(splitPane) self.is_open = True def createButtonPane(self): """Create a new button pane with the type editor window""" self._button_listener = TypeEditorButtonListener(self) panel = JPanel() panel.setLayout(BoxLayout(panel, BoxLayout.Y_AXIS)) panel.add(Box.createRigidArea(Dimension(0, 5))) panel.add( self.createButton("Validate", "validate", "Check if typedef is valid") ) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add(self.createButton("Save", "save", "Save the typedef")) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add(self.createButton("Reset", "reset", "Reset to original")) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add(self.createButton("Exit", "exit", "Close window and reset")) panel.add(Box.createRigidArea(Dimension(0, 3))) return panel def createButton(self, text, command, tooltip): """Generate a new button with a given text and command""" button = JButton(text) button.setAlignmentX(Component.CENTER_ALIGNMENT) button.setActionCommand(command) button.addActionListener(self._button_listener) button.setToolTipText(tooltip) return button def applyType(self): """Callback for the apply button. Validates the definition and calls the callback provided when opening the window """ try: new_json = self._type_editor.getText().tostring() message_type = json.loads(new_json) if self._original_json == new_json and len(message_type) != 0: # Detect if no text was changed, but allow an empty type for # adding empty ones self.exitTypeWindow() return blackboxprotobuf.validate_typedef(message_type, self._original_typedef) self._type_callback(message_type, self._type_source) self.exitTypeWindow() except Exception as exc: self._burp_callbacks.printError(traceback.format_exc()) JOptionPane.showMessageDialog(self, "Error saving type: " + str(exc)) def resetTypeWindow(self): """Callback for reset button. Resets to the original type definition""" self._type_editor.setText(json.dumps(self._original_typedef, indent=4)) def exitTypeWindow(self): """Callback for exit button. Exits the window without saving""" self.is_open = False self.dispatchEvent(WindowEvent(self, WindowEvent.WINDOW_CLOSING)) def validateType(self): """Callback for validate button. Validates the type without saving""" try: message_type = json.loads(self._type_editor.getText().tostring()) except Exception as exc: self._burp_callbacks.printError(traceback.format_exc()) JOptionPane.showMessageDialog(self, "Error decoding JSON: " + str(exc)) return try: blackboxprotobuf.validate_typedef(message_type, self._original_typedef) except Exception as exc: self._burp_callbacks.printError(traceback.format_exc()) JOptionPane.showMessageDialog(self, "Error validating type: " + str(exc)) return class TypeEditorButtonListener(ActionListener): """Button action listener for the type editor window""" def __init__(self, type_editor): self._type_editor = type_editor def actionPerformed(self, event): """Called when a button is pressed""" if event.getActionCommand() == "validate": self._type_editor.validateType() elif event.getActionCommand() == "save": self._type_editor.applyType() elif event.getActionCommand() == "reset": self._type_editor.resetTypeWindow() elif event.getActionCommand() == "exit": self._type_editor.exitTypeWindow() blackboxprotobuf-1.4.2/burp/blackboxprotobuf/burp/typedef_tab.py000066400000000000000000000363771476504373500252500ustar00rootroot00000000000000""" TypeDefinitionTab is a top-level Burp Suite tab which allows saved/named types to be added/modified at any time. It also adds options for importing/exporting protobuf types to .json files. """ # Copyright (c) 2018-2023 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import re import json import burp import traceback import blackboxprotobuf from javax.swing import JSplitPane, JScrollPane, JPanel, JButton, BoxLayout, Box from javax.swing import JOptionPane, JList, ListSelectionModel, JFileChooser from javax.swing.filechooser import FileNameExtensionFilter from java.awt import Component, Dimension from java.awt.event import ActionListener from javax.swing.border import EmptyBorder from blackboxprotobuf.lib.api import sort_typedef from blackboxprotobuf.lib.config import default as default_config from blackboxprotobuf.burp import typedef_editor # TODO put these in one place NAME_REGEX = re.compile(r"\A[a-zA-Z_][a-zA-Z0-9_]*\Z") class TypeDefinitionTab(burp.ITab): """Implements an interface for editing known message type definitions.""" def __init__(self, extension, burp_callbacks): self._burp_callbacks = burp_callbacks self._extension = extension self._type_list_component = JList(extension.known_message_model) self._type_list_component.setSelectionMode( ListSelectionModel.MULTIPLE_INTERVAL_SELECTION ) self._component = JPanel() self._component.setLayout(BoxLayout(self._component, BoxLayout.Y_AXIS)) splitPane = JSplitPane(JSplitPane.HORIZONTAL_SPLIT) splitPane.setRightComponent(JScrollPane(self._type_list_component)) splitPane.setLeftComponent(self.createButtonPane()) splitPane.setResizeWeight(0.03) splitPane.setMaximumSize(Dimension(1000, 1000)) self._component.add(splitPane) self._component.add(Box.createVerticalGlue()) self._component.setBorder(EmptyBorder(10, 10, 10, 10)) def getTabCaption(self): """Returns name on tab""" return "Protobuf Type Editor" def getUiComponent(self): """Returns Java AWT component for tab""" return self._component def createButtonPane(self): """Create AWT window panel for buttons""" self._button_listener = TypeDefinitionButtonListener(self) panel = JPanel() panel.setLayout(BoxLayout(panel, BoxLayout.Y_AXIS)) panel.add(Box.createRigidArea(Dimension(0, 5))) panel.add(self.createButton("Add", "new-type", "Create a new type")) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add(self.createButton("Edit", "edit-type", "Edit the selected type")) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add( self.createButton("Rename", "rename-type", "Rename the selected type") ) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add( self.createButton("Remove", "delete-type", "Delete all selected types") ) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add( self.createButton( "Save All Types To File", "save-types", "Save all known types as JSON to a file", ) ) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add( self.createButton( "Load All Types From File", "load-types", "Load types from JSON file" ) ) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add( self.createButton( "Export All types As .proto", "export-proto", "Export all types as .proto", ) ) panel.add(Box.createRigidArea(Dimension(0, 3))) panel.add( self.createButton( "Import .proto", "import-proto", "Import types from a .proto file" ) ) panel.add(Box.createRigidArea(Dimension(0, 3))) return panel def createButton(self, text, command, tooltip): """Generate new button with the given text and command string""" button = JButton(text) button.setAlignmentX(Component.CENTER_ALIGNMENT) button.setActionCommand(command) button.addActionListener(self._button_listener) button.setToolTipText(tooltip) return button def save_callback(self, typedef, name): """Save typedef and update list for a given message name""" if name not in default_config.known_types: self._extension.known_message_model.addElement(name) default_config.known_types[name] = typedef def add_typedef(self): type_name = JOptionPane.showInputDialog("Enter new name") # Error out if already defined if type_name in default_config.known_types: JOptionPane.showMessageDialog( self._component, 'Message type "%s" already exists' % type_name, ) return self._extension.open_typedef_editor({}, type_name, self.save_callback) def edit_typedef(self): list_component = self._type_list_component if list_component.isSelectionEmpty(): return type_name = list_component.getSelectedValue() message_type = default_config.known_types[type_name] self._extension.open_typedef_editor( sort_typedef(message_type), type_name, self.save_callback ) class TypeDefinitionButtonListener(ActionListener): """Callback listener for buttons in the TypeDefinition interface""" def __init__(self, typedef_tab): self._typedef_tab = typedef_tab def actionPerformed(self, event): """Called when a button is pressed.""" if event.getActionCommand() == "new-type": self._typedef_tab.add_typedef() elif event.getActionCommand() == "edit-type": self._typedef_tab.edit_typedef() elif event.getActionCommand() == "rename-type": list_component = self._typedef_tab._type_list_component # Check if something is selected if list_component.isSelectionEmpty(): return # Get's only the first value previous_type_name = list_component.getSelectedValue() new_type_name = JOptionPane.showInputDialog( "Enter new name for %s:" % previous_type_name ) if new_type_name in default_config.known_types: JOptionPane.showMessageDialog( self._typedef_tab._component, 'Message type "%s" already exists' % new_type_name, ) return if previous_type_name not in default_config.known_types: JOptionPane.showMessageDialog( self._typedef_tab._component, 'Message type "%s" does not exist' % previous_type_name, ) return if not NAME_REGEX.match(new_type_name): JOptionPane.showMessageDialog( self._typedef_tab._component, 'Message type name "%s" is not valid.' % new_type_name, ) return typedef = default_config.known_types[previous_type_name] default_config.known_types[new_type_name] = typedef del default_config.known_types[previous_type_name] # TODO should manage this centrally somewhere self._typedef_tab._extension.refresh_message_model() for key, typename in self._typedef_tab._extension.saved_types.items(): if typename == previous_type_name: self._typedef_tab._extension.saved_types[key] = new_type_name elif event.getActionCommand() == "delete-type": list_component = self._typedef_tab._type_list_component # Check if something is selected if list_component.isSelectionEmpty(): return type_names = list_component.getSelectedValuesList() # TODO Confirm delete? for type_name in type_names: del default_config.known_types[type_name] self._typedef_tab._extension.refresh_message_model() elif event.getActionCommand() == "save-types": chooser = JFileChooser() chooser.setFileFilter( FileNameExtensionFilter("JSON Type Definition", ["json"]) ) chooser.setMultiSelectionEnabled(False) action = chooser.showSaveDialog(self._typedef_tab.getUiComponent()) if ( action == JFileChooser.CANCEL_OPTION or action == JFileChooser.ERROR_OPTION ): return file_name = chooser.getSelectedFile().getCanonicalPath() ext = os.path.splitext(file_name)[1] if ext != ".json": # Add json extension if it doesn't have one file_name += ".json" with open(file_name, "w+") as selected_file: json.dump( default_config.known_types, selected_file, indent=4, sort_keys=True, ) elif event.getActionCommand() == "load-types": chooser = JFileChooser() chooser.setFileFilter( FileNameExtensionFilter("JSON Type Definition", ["json"]) ) chooser.setMultiSelectionEnabled(False) action = chooser.showOpenDialog(self._typedef_tab.getUiComponent()) if ( action == JFileChooser.CANCEL_OPTION or action == JFileChooser.ERROR_OPTION ): return file_name = chooser.getSelectedFile().getCanonicalPath() types = {} with open(file_name, "r") as selected_file: types = json.load(selected_file) for key, value in types.items(): # check to make sure we don't nuke existing messages if key in default_config.known_types: overwrite = ( JOptionPane.showConfirmDialog( self._typedef_tab._component, "Message %s already saved. Overwrite?" % key, ) == 0 ) if not overwrite: continue default_config.known_types[key] = value self._typedef_tab._extension.refresh_message_model() elif event.getActionCommand() == "export-proto": chooser = JFileChooser() chooser.setFileFilter( FileNameExtensionFilter("Protobuf Type Definition", ["proto"]) ) chooser.setMultiSelectionEnabled(False) action = chooser.showSaveDialog(self._typedef_tab.getUiComponent()) if ( action == JFileChooser.CANCEL_OPTION or action == JFileChooser.ERROR_OPTION ): return file_name = chooser.getSelectedFile().getCanonicalPath() ext = os.path.splitext(file_name)[1] if ext == "": # No extension, add .proto file_name += ".proto" if os.path.exists(file_name): # 0 is the YES option overwrite = ( JOptionPane.showConfirmDialog( self._typedef_tab._component, "File %s already exists. Overwrite?" % file_name, ) == 0 ) if not overwrite: return print("overwriting file: %s" % file_name) try: blackboxprotobuf.export_protofile(default_config.known_types, file_name) except Exception as exc: self._typedef_tab._burp_callbacks.printError(traceback.format_exc()) JOptionPane.showMessageDialog( self._typedef_tab._component, "Error saving .proto file: " + str(exc), ) elif event.getActionCommand() == "import-proto": chooser = JFileChooser() chooser.setFileFilter( FileNameExtensionFilter("Protobuf Type Definition", ["proto"]) ) chooser.setMultiSelectionEnabled(False) action = chooser.showOpenDialog(self._typedef_tab.getUiComponent()) if ( action == JFileChooser.CANCEL_OPTION or action == JFileChooser.ERROR_OPTION ): return file_name = chooser.getSelectedFile().getCanonicalPath() if not os.path.exists(file_name): self._typedef_tab._burp_callbacks.printError( "Attempted to import %s, but the file does not exist." % file_name ) JOptionPane.showMessageDialog( self._typedef_tab._component, "File %s does not exist to import." + str(exc), ) return try: new_typedefs = blackboxprotobuf.import_protofile( file_name, save_to_known=False ) for key, value in new_typedefs.items(): # check to make sure we don't nuke existing messages if key in default_config.known_types: overwrite = ( JOptionPane.showConfirmDialog( self._typedef_tab._component, "Message %s already saved. Overwrite?" % key, ) == 0 ) if not overwrite: continue else: self._typedef_tab._extension.known_message_model.addElement(key) default_config.known_types[key] = value except Exception as exc: self._typedef_tab._burp_callbacks.printError(traceback.format_exc()) JOptionPane.showMessageDialog( self._typedef_tab._component, "Error loading .proto file: " + str(exc), ) blackboxprotobuf-1.4.2/burp/blackboxprotobuf/burp/user_funcs.py000066400000000000000000000120661476504373500251230ustar00rootroot00000000000000"""These functions provide hooks into the blackboxprotobuf Burp extension, allowing certain functionality to be customized in order to handle non-standard applications. """ # Copyright (c) 2018-2023 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # Documentation for Common parameters: # content -- Binary content of the request # is_request -- boolean, True for a request, False for a response # content_info -- RequestInfo or ResponseInfo object, See # https://portswigger.net/Burp/extender/api/burp/IRequestInfo.html and # https://portswigger.net/Burp/extender/api/burp/IResponseInfo.html # helpers -- Burp extension helpers, # https://portswigger.net/Burp/extender/api/burp/IExtensionHelpers.html # request/request_content_info -- If called on a response, send the # corresponding request (useful for retrieving URL parameters) # Useful functionality: # URL parameters: # for param in content_info.getParmeters(): # if param.getName() == 'type': # ... # Headers: # if 'content-type' in content_info.getHeaders(): # ... # Request Body: # body = content[content_info.getBodyOffset():].tostring() # Setting paramater: # import burp.IParameter # body = helpers.updateParameter( # content, # helpers.buildParameter('message', # protobuf_data, # IParameter.PARAM_URL)) def detect_protobuf(content, is_request, content_info, helpers): """Customize protobuf detection with a request or a response. Should return True if it is a protobuf, False if it definitely not a protobuf and should not try to decode, or None to fallback to the standard content-type header detection. This can be used to add protobuf detection based on different headers or customer application functionality. You can also use "return True" to try to decode every request/response. """ pass def get_protobuf_data( content, is_request, content_info, helpers, request=None, request_content_info=None ): """Customize how the protobuf data is retrieved from the request. By default, it is assumed the body of the request/response contains a protobuf payload and tries to detect if it's compressed. This function can be used if the payload is in a non-standard location or has a non-standard encoding. It should return the raw protobuf bytes from the message. """ pass def set_protobuf_data( protobuf_data, content, is_request, content_info, helpers, request=None, request_content_info=None, ): """Customize how the protobuf data is set in request/response. This function is used to configure where in the request the protobuf data is placed and how it is encoded if the message is modified. For example, this could set a query parameter with base64 encoded data if that is what the application expects. This function is the mirror to `get_protobuf_data` and if one function is modified, then the other should as well. This is expected to return bytes representing the HTTP headers and body, such as that returned by the `buildHttpMessage` function of the Burp helpers (https://portswigger.net/burp/extender/api/burp/iextensionhelpers.html#buildHttpMessage) """ pass def hash_message( content, is_request, content_info, helpers, request=None, request_content_info=None ): """Blackbox protobuf remembers which type definition to use for each request/response by saving them in a dictionary, keyed by a 'message hash'. If a message has the same hash, it will try to re-use the same type definition. By default, this is the request path and true or false depending whether it is a request or response. If there isn't a one-to-one mapping of message types to URLs, then this function can be used to customize what parts of the request to use for identification (eg. a message type header or parameter). This function should return a string. """ pass blackboxprotobuf-1.4.2/burp/deps/000077500000000000000000000000001476504373500170055ustar00rootroot00000000000000blackboxprotobuf-1.4.2/burp/deps/six/000077500000000000000000000000001476504373500176105ustar00rootroot00000000000000blackboxprotobuf-1.4.2/docs/000077500000000000000000000000001476504373500160325ustar00rootroot00000000000000blackboxprotobuf-1.4.2/docs/TypeDefs.md000066400000000000000000000413751476504373500201110ustar00rootroot00000000000000# Type Definitions Type Definitions (also referred to as typedefs) are the data structure the BBPB uses to remember metadata about a message and it's fields. It contains the exact type for each field and other metadata such as the field name. ## Decoding The typedef is *optional* when decoding a message. Field types and names will be used during decoding if they are provided, but if there is no typedef or the typedef doesn't contain metadata for a field, BBPB will choose a default type based on the field's "wire type" (see Wire Types) below. The default type guessed by BBPB during decoding may not match the actual type and is simply a best effort guess. The user may need to use the typedef to change the type to the correct value. The decoding function will return a type definition which contains what types it used to decode each field number in the message. These types are required to be able to re-encode the message. If a typedef was provided to the decoder, the decoding function returns a copy of the typedef and adds type values for any unknown fields. ## Encoding The typedef is *required* when encoding a message back to a protobuf message. It contains the data necessary for the encoder to map each message field to the correct binary representation. If the message has a field that is not in the typedef, it will raise an exception. In most cases, the encoder should be provided with the exact type definition returned by the decoder. Modifying the type definition before encoding could lead to unexpected errors with data type mismatches and inconsistent encoding. If you wish to change the type of a field in the message, the message should be decoded again with the modified typedef. ## Type Definition Format A type definition is a Python dictionary/JSON object where each key is the field number and the value is metadata about the field (called the Field Definition). For example: ~~~ { "1": { "name": "email", "type": "string", }, "2": { "name": "uid", "type": "int", }, "3": { "type": "string", } } ~~~ The field definition may contain the following fields: ### `type` (Required) The type field is a required field which should contain the BBPB type for this field. These types map roughly to protobuf types, but may not be an exact match. The following types are valid BBPB types: - `uint` - `int` - `sint` - `fixed32` - `sfixed32` - `float` - `fixed64` - `sfixed64` - `double` - `bytes` - `bytes_hex` - `string` - `message` - `packed_uint` - `packed_int` - `packed_sint` - `packed_fixed32` - `packed_sfixed32` - `packed_float` - `packed_fixed64` - `packed_sfixed64` - `packed_double` The latest list of types can also be found in [type_maps.py](/lib/blackboxprotobuf/lib/types/type_maps.py). ### `name` User friendly name for the field which can be used instead of the field number in decoded messages. If the name is an empty string (`""`), it is ignored. ### `message_typedef` Either `message_typedef` or `message_type_name` are required if `type` is `message`. A type of `message` indicates that this field contains an sub-message. This field contains the type definition used to encode/decode the sub-message. ### `message_type_name` This field is an alternative to `message_typedef` and should reference another type definition in `config.known_types`. The type definitions in `known_types` will be used for this field instead of an embedded type definition. This can greatly simplify the type definition and allow for type definitions to be reused between different messages. ### `alt_typedefs` The `alt_typedefs` field is a dictionary which contains "alternative" types/type definitions for a `message` type field. Generally, protobuf does not allow a single field number to have different types, whether within the same message or across multiple messages of the same type. However, a common pattern is to embed another protobuf message using the `bytes` type instead of specific message type. This pattern is promoted by the `google.protobuf.Any` type, which is described at . This means that two protobuf messages encoded with the same message type could have embedded messages with entirely different types, and BBPB does not have a way to determine which type was used or predict the type of future messages. The "technically correct" way to handle these messages is to set the type to `bytes` and not decode the embedded message, but that's not very useful. Instead, if BBPB already has a `message_typedef` for a particular field and that `message_typedef` is not valid to decode an instance of the field, it will try to use any type definitions in `alt_typedefs`. If no type definitions in `alt_typedefs` work, BBPB will add a new type definition to `alt_typedefs`. Alternate type definitions are numbered starting at `1`, where `0` generally refers to the primary type definition stored in `message_typedef`. In the decoder output, the alternate type definition number is placed after a `-` in the field name or number to indicate which definition was used to decode the field, and therefore which definition should be used to re-encode it. For example a message with `{"1-2": { ... }}` indicates that field number `1` was decoded with `alt_typedefs[2]`. The field name can also be used instead of the field number, such as `{"user_profile-2": { ... }}`. In some rare cases, the `alt_typedef` dictionary can contain a string with a type (such as `string` or `bytes`) instead of a type definition. This should only happen if a field contained a decodable message when the typedef was created, but BBPB did not recognize a valid protobuf message on a subsequent run. ### `field_order` Field order contains the order in which fields were decoded in a protobuf sub-message. This helps BBPB avoid accidentally mutating byte fields that are mistakenly decoded as messages, but should never be required for a legitimate protobuf message. This field can be disabled by setting `config.preserve_field_order` to `False`. ### `example_value_ignored` This field may contain an example value from the decoded protobuf message. The field is only populated to help identify the correct field number in the typedef to edit and is ignored during encoding and decoding. ## Wire Types The protobuf binary does not contain exact type information for data fields. Each encoded field has a field number and a "wire type". The wire type tells the decoder how to determine the length of the field. This is important for backwards compatibility, because it allows the decoder to skip fields that it doesn't recognize and still parse the rest of the message. For more information see . Protobuf defines the following wire types: - Varint: Variable length integer representation, one bit of each byte is used to signal if it is the last byte - Fixed 64 bit: 64 bit number, could be an integer or a floating point number - Fixed 32 bit: 32 bit number, could be an integer or a floating point number - Length Delimited: Field data is prefixed with a varint containing the data length - Start/End Group: Not a field itself, but groups together fields between the tags. Deprecated in favor of sub-messages and not supported by BBPB. Based on the wiretype, BBPB can make a guess at the correct type for each field, and also ensure that the field can be re-encoded back to valid protobuf. ## Modifying TypeDefs ### Changing Field Types The type of an individual field can be corrected by modifying the "type" field in the typedef using the modified typedef to decode the protobuf data again. Avoid using the modified typedef directly with the encoder function (instead of decoding again), as this could produce inconsistent protobuf data. The modified type must have the same wiretype as the original type in the typedef, otherwise the typedef will be invalid for the original protobuf data. For example, you can change an `int` value to `sint` to use zigzag encoding, but cannot change it to `float` or `double`. #### Varint Types The varint wire type is a variable length encoding for integers. One bit of each byte is used to indicate whether there is another byte or if this is the final byte. Varints can map to several BBPB types: - `int` (default) - can represent positive and negative values - negative values are represented using two's complement, which means that the highest bit must be set for negative numbers - two's complement requires the largest possible number of varint bytes to represent negative numbers, making it inefficient if negative values are common - this is the default because it can represent most unsigned integers correctly, even if the original type is `uint` - `uint` - unsigned integers, can represent larger values than `int` - Choose this type if BBPB is decoding to negative numbers when they should always be positive - `sint` - can represent positive and negative values through Zigzag encoding - Zigzag encoding maps unsigned integers to signed integers by switching between positive and negative values. For example, 0 -> 0, 1 -> -1, 2 -> +1, 3 -> -2, ... - Represents small negative numbers more efficiently than `int`, as smaller values such as `-1` will only requires a single byte to encode instead of the varint maximum - Choose this type if the decoded numbers are off by ~ 2x from expected values or numbers are being decoded as positive when they should be negative Boolean values do not have a dedicated type in BBPB, but are encoded as a varint with a value of 0 for False or 1 for True. See and for more information on varint encoding. #### Fixed 64 Types Fixed 64 wiretype indicates that the field is always 64 bits. This can be used to represent floating point numbers and either signed or unsigned integers. Valid types are: - `fixed64` (default) - unsigned 64 bit integer - `sfixed64` - signed 64 bit integer - `double` - 64 bit floating point number The default value is an integer value. `double` might make more sense as a default choice, but will require further research before changing globally. See "Changing Default Types" for how to change the default on a per-project basis #### Fixed 32 Types Fixed 32 wiretype indicates that the field is always 32 bits. This can be used to represent floating point numbers and either signed or unsigned integers. Valid types are: - `fixed32` (default) - unsigned 32 bit integer - `sfixed32` - signed 32 bit integer - `float` - 32 bit floating point number The default value is an integer value. `float` might make more sense as a default choice, but will require further research before changing globally. See "Changing Default Types" for how to change the default on a per-project basis #### Length Delimited Types The length delimited wiretype means that the field starts with a varint representing the length of the field in bytes. This wiretype is the broadest and can represent a wide variety of field types, including embedded messages, strings, bytes and packed fields. Instead of a single default type for length delimited fields, BBPB will first try to decode the field as a `message`, then fallback to `string` if decoding fails, and finally fall back to `bytes` if `string` fails. Valid types are: - `message` - bytes represent an encoded protobuf message - The type definition for the embedded message is stored in the `message_typedef` field. - Alternatively, the `message_type_name`field can be used to reference another named typedef without embedding the entire typedef. - `bytes` - parse field bytes directly as python bytes, can represent any length delimited field - `bytes_hex` - same as bytes, but encode to a string using hex - `string` - represents UTF-8 or ASCII strings - `packed_*` - Packed fields are a more efficient mechanism for representing repeated values, such as a list or array, by removing the need for repeated tags (varint containing field number and wiretype) when encoding. - The following packed types are supported by BBPB: - `packed_uint` - `packed_int` - `packed_sint` - `packed_fixed32` - `packed_sfixed32` - `packed_float` - `packed_fixed64` - `packed_sfixed64` - `packed_double` - BBPB does not have a mechanism for detecting packed fields and these types must be set explicitly by the user. See for more information on length delimited encoding. ### Changing Default Types While editing the type definition can be used to change the types of existing fields, it's also possible to change the default type used for a particular wire type. Changing the default type must follow the same rule as type editing: the wiretype on the new type must be the same as the original wiretype. The default type for a wiretype can be modified by providing a `Config` object to the decoder and overriding the default type using the `default_types` dictionary (in [config.py](/lib/blackboxprotobuf/lib/config.py#L49>)). The key for the dictionary is the wiretype, which can be found in [wiretypes.py](/lib/blackboxprotobuf/lib/types/wiretypes.py) and match the wiretypes at . For example: ~~~ config.default_types[wiretypes.FIXED64] = 'double' ~~~ The `default_types` field in `Config` is not used for length delimited types, because of the special fallback logic for length delimited fields. For these fields, you can replace the `bytes` fallback type by changing `default_binary_type` on the [`Config` object]( /lib/blackboxprotobuf/lib/config.py#L45>). However, unlike default types, the decoder will still try to decode the field as a message or string before falling back to `default_binary_type`. The default binary type is primarily intended to allow changing the binary representation, such as `bytes_hex` instead of `bytes`, but could also be used to default to trying a packed type. ### Naming Fields By default, the decoded message will use the field number as a dictionary key. However, users can add a more readable name to the field by modifying the "name" field. The "name" field will be used as the dictionary key in the decoded message. The encoder will accept either the original field number or the name specified in the "name" field of the typedef. The field name needs to be unique and should be alphanumeric with underscores, but cannot start with a number (see regex in [api.py](/lib/blackboxprotobuf/lib/api.py#L310)). ### Creating Type Definitions And Fields From Scratch While many workflows with BBPB are designed around editing a type definition generated by the decoder, it is still possible to manually add new fields or create new type definitions from scratch. To add a new protobuf field to an existing typedef, simply add a new key to the typedef dictionary (either top level dictionary or within a `message_typedef` dictionary) and add the appropriate `type` field. The `name` field is optional, but will greatly increase readability. For example: ~~~ { "1": { ... (existing field number) ... }, "5": { "name": "uid", "type": "int" } } ~~~ or: ~~~ typedef["5"] = { "name": "uid", "type": "int } ~~~ If the `type` is `message`, then either `message_typedef` or `message_type_name` is also required. `message_typedef` should contain a dictionary representing the type definition for that field. If you know the fields of the protobuf from another source, it's possible to create type definitions from scratch. Simply create a dictionary for the top level message type definition where each key is a string containing the field number and the value of each entry is a dictionary containing the field information. Each protobuf field is required to have a `type` attribute containing the BBPB type. The `name` attribute is optional but recommended for readability. ### Cleaning Up Typedefs If a type definition will be saved for re-use or embedded in code, it can be cleaned up to improve readability and minimize the size of the definition. Any field number which has not been modified from the default value can be removed from the definition or from any `message_typedefs`. To remove a field, remove the dictionary key and associated value from the typedef dictionary. All remaining field numbers are required to have a `type` attribute, and any `message` fields need `message_typedef` or `message_type_name`. The `name` attribute is not required and can be removed if empty, but is recommended otherwise. The `alt_typedef` field can be removed if the definitions have not been customized. Any other attributes on the typedef can be removed, including: - `example_value_ignored` - `field_order` blackboxprotobuf-1.4.2/lib/000077500000000000000000000000001476504373500156505ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/CLI.md000066400000000000000000000117101476504373500166010ustar00rootroot00000000000000# Blackbox Protobuf Command Line Interface (CLI) ## Description The Blackbox Protobuf library has an embedded CLI interface which can be invoked with `python -m blackboxprotobuf` for use in shell scripts, to plug in to other tools, or easily decode arbitrary protobuf messages. ## Installation The Blackbox Protobuf library can be installed with: ~~~ pip install bbpb ~~~ The command line interface can then be run with: ~~~ bbpb ~~~ or ~~~ python3 -m blackboxprotobuf ~~~ ## Usage ### Examples Simple Decoder: ~~~ cat test_data | bbpb -r ~~~ Save type for editing: ~~~ cat test_data | bbpb -ot ./saved_type.json ~~~ Decode with type: ~~~ cat test_data | bbpb -it ./saved_type.json ~~~ Decode edit and re-encode: ~~~ cat test_data | bbpb > message.json vim message.json cat message.json | bbpb -e > test_data_out ~~~ ### Decoding The CLI decoding mode (default) will take a protobuf payload and an option type defintion, and output a JSON object which contains the decoded message and a type definition. By default, the binary protobuf message is expected to be provided on stdin. The input type cannot be provided through stdin and must be saved to file and provided via the `-it`/`--input-type` argument. Alternatively, the `-j`/`--json-protobuf` argument allows the protobuf message and typedef to be pass in as a single JSON object. The input JSON object should have a `protobuf_data` field which contains the base64 encoded protobuf data, and can optionally have a `typedef` field with the input type definition. This option is useful for tools calling the CLI which may not want to save files to disk for input types. The default output from the decoder will be a JSON object which contains the decoded message in the `message` field and the typedef necessary to decode the message in the `typedef` field. The output format matches the expected input for the CLI encoder, allowing the message to be easily edited and re-encoded. Alternatively, the `-r`/`--raw-decode` argument will provide a simpler output with just the JSON message and no type definition. This is useful if you don't want to edit the message, just view it, or are saving the type definition to a file with `-ot`/`--output-type` argument. The `-it`/`--input-type` and `-ot`/`--output-type` arguments will have the CLI read and/or write type definitions to the provided file. ### Encoding The `-e`/`--encode` argument put the CLI in encoding mode, which takes a JSON message type definition, and prints an encoded protobuf message to stdin. By default, the CLI expects a JSON object through stdin which contains a `message` field with the JSON representation of the message and a `typedef` field with the type definition. This format should match the output of the CLI decoder. The type definition can also be provided through a file specified with the `-it`/`--input-type` argument. If the type definition is provided through this argument and there is no `message` field on the input JSON, the encoder will use the entire input JSON as the message (eg. the output of the decoder with `-r`/`--raw-decode`). By default, the CLI will output the encoded protobuf bytes to stdout. Alternatively, the `-j`/`--json-protobuf` command line flag will output a JSON payload with `protobuf_data` and `typedef` attributes. The protobuf data field will contain base64 encoded protobuf data. This format matches the expected input of the decoder with the `-j`/`--json-protobuf` attribute. ### Editing The messages and typedefs can be easily edited following the same rules as other Blackbox Protobuf interfaces. The JSON message from the decoder can be edited to easily change field values, before passing the payload back to the encoder. It is possible to add fields if the field type is defined in the type definition and the added value matches the type definition. If you wish to edit the type definition to change field names or types, save the type definition from the output payload or the `-ot`/`--output-typedef` argument. Edit the type definition and then perform the decoding step again with `-it`/`--input-typedef`. It is not recommended that you edit the typedef from the decoder directly before passing the message/typedef to the encoder, as this may cause the payload to be encoded incorrectly. ### Payload Encoding The Blackbox Protobuf library tries to automatically handle several "wrapper" encodings. The library currently supports gzip compression and gRPC headers. During decoding, the library will attempt to detect these wrappers and unpack the protobuf payload. If a payload encoding is identified, it is stored in `payload_encoding` field of the output JSON. The encoder will then re-apply the wrapper when the payload is encoded. If the payload encoding is not provided, the encoder will default to "none" which indicates plain protobuf. The payload encoding is set to "gzip" or "grpc" for other encoding options. The payload encoding process can be overridden during decoding or encoding with the `-pe`/`--payload-encoding` argument. blackboxprotobuf-1.4.2/lib/Makefile000066400000000000000000000012141476504373500173060ustar00rootroot00000000000000all: check test_py2 test_py3 setup_py2: rm -rf /tmp/bbpb-python2-tmp python2 -m virtualenv /tmp/bbpb-python2-tmp . /tmp/bbpb-python2-tmp/bin/activate python2 -m pip install -r tests/requirements-python2-dev.txt test_py2: setup_py2 python2 -m pytest tests/py_test/ test_py3: poetry env use python3 poetry run python -m pytest tests/py_test/ format: poetry run black . check: poetry run mypy --strict blackboxprotobuf/ prepublish: poetry build poetry config repositories.test-pypi https://test.pypi.org/legacy/ # poetry config pypi-token.test-pypi TOKEN # poetry publish -r test-pypi # poetry config pypi-token.pypi TOKEN # poetry publish blackboxprotobuf-1.4.2/lib/README.md000066400000000000000000000252351476504373500171360ustar00rootroot00000000000000# BlackBox Protobuf Library ## Description Blackbox protobuf library is a Python module for decoding and re-encoding protobuf messages without access to the source protobuf descriptor file. This library provides a simple Python interface to encode/decode messages that can be integrated into other tools. This library is targeted towards use in penetration testing where being able to modify messages is critical and a protocol buffer definition may not be readily available. ## Background Protocol Buffers (protobufs) are a standard published by Google with accompanying libraries for binary serialization of data. Protocol buffers are defined by a `.proto` file known to both the sender and the receiver. The actual binary message does not contain information such as field names or most type information. For each field, the serialized protocol buffer includes two pieces of metadata, a field number and the wire type. The wire type tells a parser how to parse the length of the field, so that it can be skipped if it is not known (one protocol buffer design goal is being able to handle messages with unknown fields). A single wire-type generally encompasses multiple protocol buffer types, for example the length delimited wire-type can be used for string, bytestring, inner message or packed repeated fields. See for the breakdown of wire types. The protocol buffer compiler (`protoc`) does support a similar method of decoding protocol buffers without the definition with the `--decode_raw` option. However, it does not provide any functionality to re-encode the decoded message. ## How it works The library makes a best effort guess of the type based on the provided wire type (and occasionally field content) and builds a type definition that can be used to re-encode the data. In general, most fields of interest are likely to be parsed into a usable form. Users can optionally pass in custom type definitions that override the guessed type. Custom type definitions also allow naming of fields to improve user friendliness. # Usage ## Installation The package can be installed from source with: ``` poetry install ``` BlackBox Protobuf is also available on PyPi at . It can be installed with: ``` pip install bbpb ``` ## CLI The package defines a `bbpb` command for command line encoding/decoding. For command line usage see [CLI.md](./CLI.md). ## Interface The main `blackboxprotobuf` module defines an API with the core encode/decode message functions, along with several convenience functions to make it easier to use blackboxprotobuf with a user interface, such as encoding/decoding directly to JSON and validating modified type definitions. ### Decode Decoding functions takes a protobuf bytestring, and optionally either a type definition or a known message name mapped to a type definition (in `blackboxprotobuf.known_messages`). If a type definition isn't provided, an empty message type is assumed and all types are derived from the protobuf binary. The decoder returns a tuple containing a dictionary with the decoded data and a dictionary containing the generated type definition. If the input type definition does not include types for all fields in the message, the output type definitions will include type guesses for those fields. Example use: ```python import blackboxprotobuf import base64 data = base64.b64decode('KglNb2RpZnkgTWU=') message,typedef = blackboxprotobuf.protobuf_to_json(data) print(message) ``` ### Encode The encoding functions takes a Python dictionary containing the data and a type definition. Unlike decoding, the type definition is required and will fail if any fields are not defined. Generally, the type definition should be the output from the decoding function or a modified version thereof. Example use: ```python import blackboxprotobuf import base64 data = base64.b64decode('KglNb2RpZnkgTWU=') message,typedef = blackboxprotobuf.decode_message(data) message[5] = 'Modified Me' data = blackboxprotobuf.encode_message(message,typedef) print(data) ``` ### Type definition structure The type definition object is a Python dictionary representing the type structure of a message, it includes a type for each field and optionally a name. Each entry in the dictionary represents a field in the message. The key should be the field number and the value is a dictionary containing attributes. At the minimum the dictionary should contain the 'type' entry which contains a string identifier for the type. Valid type identifiers can be found in `blackboxprotobuf/lib/types/type_maps.py`. Message fields will also contain one of two entries, 'message_typedef' or 'message_type_name'. 'message_typedef' should contain a second type definition structure for the inner message. 'message_type_name' should contain the string identifier for a message type previously stored in `blackboxprotobuf.known_messages`. If both are specified, the 'message_type_name' will be ignored. ### JSON Encode/Decode The `protobuf_to_json` and `protobuf_from_json` functions are convenience functions for encoding/decoding messages to JSON instead of a python dictionary. These functions are designed for user-facing input/output and will also automatically sort the output, try to encode bytestrings for better printing and annotate example values onto the type definition structure. ### Export/import protofile The `export_protofile` and `import_protofile` will attempt to convert a protobuffer `.proto` file into the blackboxprotobuf type definition and vice versa. These functions provide a higher level interface to `blackboxprotobuf.lib.protofile` which only takes a filename. The protofile functions do not implement a full proper parser and may break on some types. One common case to be aware of is the "import" statements in ".proto" files, which are not supported. Any imported files must be manually imported with `import_protofile` and saved in `blackboxprotobuf.known_messages` first. ### Validate Typedef The `validate_typedef` function is designed to sanity check modified type definitions and make sure they are internally consistent and consistent with the previous type definition (if provided). This should help catch issues such as changing a field to an incompatible type or duplicate field names. ### Output Helper Functions The `json_safe_transform` is a helper function to help create more readable JSON output of bytes. It will encode/decode bytes types as `latin1` based on the type in the type definition. The `sort_output` is a helper function which sorts the output message based on the field numbers from the typedef. This helps makes the JSON output more consistent and predictable. The `sort_typedef` function sorts the fields of the typedef in order to make the output more readable. The message fields are sorted by their number and type fields (eg. name, type, inner message typedef) are sorted to prioritize important short fields at the top and especially to keep the name and type fields from getting buried underneath a long inner typedef. ### Config Many of the functions accept a `config` keyword argument of the `blackboxprotobuf.lib.config.Config` class. The config object allows modifying some of the encoding/decoding functionality and storing some state. This replaces some variables that were global before. At the moment this includes: * `known_types` - Mapping of message type names to typedef (previously `blackboxprotobuf.known_messages`) * `default_binary_type` - Change the default type choice for binary fields when decoding previously unknown fields. Defaults to `bytes` but can be set to `bytes_hex` to return a hex encoded string instead. `bytes_base64` might be another option in the future. The type can always be changed for an individual field by changing the `type` in the typedef. * `default_types` - Change the default type choice for any wiretype when decoding a previously unknown field. For example, to default to unsigned integers for all varints, set `default_types[WIRETYPE_VARINT] = 'uint'`. The `api` functions like `blackboxprotobuf.decode_message` will default to using the global `blackboxprotobuf.lib.config.default` object if one is not specified. ## Type Breakdown The following is a quick breakdown of wire types and default values. See for more detailed information from Google. ### Variable Length Integers (varint) The `varint` wire type represents integers with multiple bytes where one bit of each is dedicated to indicating if it is the last byte. This can be used to represent integers (signed/unsigned), boolean values or enums. Integers can be encoded using three variations: - `uint`: Varint encoding with no representation of negative numbers. - `int`: Standard encoding but inefficient for negative numbers (always 10 bytes). - `sint`: Uses ZigZag encoding to efficiently represent negative numbers by mapping negative numbers into the integer space. For example -1 is converted to 1, 1 to 2, -2 to 3, and so on. This can result in drastically different numbers if a type is misinterpreted and either the original or incorrect type is `sint`. The default is currently `int` with no ZigZag encoding. ### Fixed32/64 The fixed length wire types have an implicit size based on the wire type. These support either fixed size integers (signed/unsigned) or fixed size floating point numbers (float/double). The default type for these is the floating point type as most integers are more likely to be represented by a varint. ### Length Delimited Length delimited wire types are prefixed with a `varint` indicating the length. This is used for strings, bytestrings, inner messages and packed repeated fields. Messages can generally be identified by validating if it is a valid protobuf binary. If it is not a message, the default type is a string/byte which are relatively interchangeable in Python. A different default type (such as `bytes_hex`) can be specified by changing `blackboxprotobuf.lib.types.default_binary_type`. Packed repeated fields are arrays of either `varints` or a fixed length wire type. Non-packed repeated fields use a separate tag (wire type + field number) for each element, allowing them to be easily identified and parsed. However, packed repeated fields only have the initial length delimited wire type tag. The parser is assumed to know the full type already for parsing out the individual elements. This makes this field type difficult to differentiate from an arbitrary byte string and will require user intervention to identify. In protobuf version 2, repeated fields had to be explicitly declared packed in the definition. In protobuf version 3, repeated fields are packed by default and are likely to become more common. blackboxprotobuf-1.4.2/lib/blackboxprotobuf/000077500000000000000000000000001476504373500212165ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/blackboxprotobuf/__init__.py000066400000000000000000000030511476504373500233260ustar00rootroot00000000000000"""`blackboxprotobuf` provides APIs for decoding and encoding binary protobuf messages. This module re-exports the functions defined in `blackboxprotobuf.lib.api`, which provides a high level interface and convenience functions. This module is split into two submodules: - The `lib` submodule contains the decoding/encoding logic and provides a python interface. - The `burp` submodule defines the Burp Suite plugin and UI. """ # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from blackboxprotobuf.lib.api import * blackboxprotobuf-1.4.2/lib/blackboxprotobuf/__main__.py000066400000000000000000000253271476504373500233210ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import sys import json import base64 import argparse import typing from typing import Any, Dict, Optional, Tuple from .lib.exceptions import BlackboxProtobufException from .lib import api from .lib import payloads from .lib.pytypes import TypeDefDict, Message def main(): # type: () -> int parser = argparse.ArgumentParser(description="Decode/Encode Protobuf Messages") parser.add_argument( "-e", "--encode", action="store_true", help="Switch to encode mode. Command decodes by default", ) parser.add_argument( "-j", "--json-protobuf", action="store_true", help='Use JSON objects which may contain base64\'d protobuf and typedefs instead of just raw protobuf bytes for encoding input and decoding output. The JSON may have the following keys: "message", "typedef", "payload_encoding".', ) parser.add_argument( "--compact", action="store_true", help="Use compact/non-pretty JSON output", ) parser.add_argument( "-pe", "--payload-encoding", action="store", help="Override the wrapper encoding for the payload, such as gzip or grpc", ) parser.add_argument( "-it", "--input-type", action="store", help="File to read the typedef from instead of stdin", ) parser.add_argument( "-ot", "--output-type", action="store", help='(Decoding) File to write the typedef to instead of stdout. This file may be a plain JSON typedef or a json object with "typedef" and "payload_encoding" fields.', ) parser.add_argument( "-r", "--raw-decode", action="store_true", help="(Decoding) Output just the decoded JSON and no type information.", ) args = parser.parse_args() message = None # type: str | bytes | Message | None typedef = None # type: TypeDefDict | None payload_encoding = None # type: str | None if args.input_type: typedef, payload_encoding = _read_input_typedef_arg(args) if args.payload_encoding: payload_encoding = args.payload_encoding input_data = _read_input(args) if args.encode: input_json = json.loads(input_data) message, typedef, payload_encoding = _read_input_json_encoding( args, input_json, typedef, payload_encoding ) elif args.json_protobuf: input_json = json.loads(input_data) message, typedef, payload_encoding = _read_input_json_decoding( args, input_json, typedef, payload_encoding ) else: message = input_data if payload_encoding is None: payload_encoding = "none" # Start basic, raw protobuf decoding if args.encode: if not isinstance(message, dict): sys.stderr.write("Error did not get a valid message to encode") return 1 if typedef is None: sys.stderr.write("Encoding requires a valid type definition") return 1 return _encode(args, message, typedef, payload_encoding) else: if isinstance(message, str): message = message.encode("utf-8") if not isinstance(message, bytes): sys.stderr.write("Error did not get a valid message to decode") return 1 return _decode(args, message, typedef, payload_encoding) # Reads input from the location from args # Does not handle any JSON decoding def _read_input(args): # type: (argparse.Namespace) -> str | bytes if args.encode or args.json_protobuf: # Text return sys.stdin.read() else: # Binary return sys.stdin.buffer.read() # Writes output to the location from args # Does not handle any JSON encoding def _write_output(args, data): # type: (argparse.Namespace, str | bytes) -> None if isinstance(data, str): # Text sys.stdout.write(data) else: # Binary sys.stdout.buffer.write(data) def _read_input_typedef_arg(args): # type: (argparse.Namespace) -> Tuple[TypeDefDict, Optional[str]] with open(args.input_type, "r") as f: input_json = json.load(f) if "typedef" in input_json: return input_json.get("typedef"), input_json.get("payload_encoding") else: # Return whole paylaod as typedef, no encoding return input_json, None def _write_output_typedef_arg(args, typedef): # type: (argparse.Namespace, Dict[str, str | TypeDefDict]) -> None with open(args.output_type, "w") as f: f.write(_to_json(args, typedef)) def _to_json(args, data): # type: (argparse.Namespace, Dict[str, Any]) -> str if args.compact: return json.dumps(data) else: return json.dumps(data, indent=2) def _read_input_json_encoding(args, input_json, typedef, payload_encoding): # type: (argparse.Namespace, Message | Dict[str, str | Message | TypeDefDict], Optional[TypeDefDict], Optional[str]) -> Tuple[Message, Optional[TypeDefDict], Optional[str]] if typedef is None and "typedef" not in input_json: sys.stderr.write( "Error: Did not get a typedef from --input-type or stdin. A typedef is required for encoding\n" ) sys.exit(1) message = typing.cast(Message | None, input_json.get("message")) if message is None: # Whole input is message. We already checked to make sure we have a typedef, so we can ditch it return typing.cast(Message, input_json), typedef, None if typedef is None: typedef = typing.cast(TypeDefDict | None, input_json.get("typedef")) if payload_encoding is None: json_payload_encoding = input_json.get("payload_encoding") if isinstance(json_payload_encoding, str): payload_encoding = json_payload_encoding elif json_payload_encoding is None: payload_encoding = None else: sys.stderr.write( "Warn: Payload encoding must be a string value: %r" % payload_encoding ) payload_encoding = None return message, typedef, payload_encoding def _read_input_json_decoding(args, input_json, typedef, payload_encoding): # type: (argparse.Namespace, Dict[str, TypeDefDict | str], Optional[TypeDefDict], Optional[str]) -> Tuple[bytes, Optional[TypeDefDict], Optional[str]] # Return message, typedef, payload_encoding message = typing.cast(str | None, input_json.get("protobuf_data")) if message is None: sys.stderr.write('Error: Did not get a "protobuf_data" attribute in input JSON') sys.exit(1) if typedef is None: typedef = typing.cast(TypeDefDict | None, input_json.get("typedef")) if payload_encoding is None: payload_encoding = typing.cast(str | None, input_json.get("payload_encoding")) # base64 decode if protobuf_json when decoding protobuf_data = base64.b64decode(message) return protobuf_data, typedef, payload_encoding def _encode(args, message, typedef, payload_encoding): # type: (argparse.Namespace, Message, TypeDefDict, Optional[str]) -> int if typedef is None: sys.stderr.write("Error: Cannot encode without a valid typedef") return 1 if not payload_encoding: payload_encoding = "none" # Re jsonify so that bbpb can fix bytes message_json = json.dumps(message) protobuf_data = api.protobuf_from_json(message_json, typedef) data = payloads.encode_payload(protobuf_data, payload_encoding) if args.json_protobuf: json_out = { "protobuf_data": base64.b64encode(data).decode("ascii"), "typedef": typedef, # Typedef is a bit redundant here } if payload_encoding != "none": json_out["payload_encoding"] = payload_encoding _write_output(args, _to_json(args, json_out)) else: _write_output(args, data) return 0 def _decode(args, data, typedef, payload_encoding): # type: (argparse.Namespace, bytes, Optional[TypeDefDict], str) -> int if len(data) == 0: sys.stderr.write("Error: Input data cannot be empty\n") return 1 # args.protobuf_json is already handled if payload_encoding: # Use provided payload encoding algorithm protobuf_data, payload_encoding = payloads.decode_payload( data, payload_encoding ) message_json, output_typedef = api.protobuf_to_json(protobuf_data, typedef) else: # Have to guess the decoding algorithm decoders = payloads.find_decoders(data) for decode in decoders: try: protobuf_data, encoding_alg = decode(data) except BlackboxProtobufException: # The "none" algorithm should always succeed continue try: message_json, output_typedef = api.protobuf_to_json( protobuf_data, typedef ) break except BlackboxProtobufException as exc: if encoding_alg == "none": raise exc message = json.loads(message_json) if args.output_type: output_typedef_data = {} # type: Dict[str, TypeDefDict | str] output_typedef_data["typedef"] = output_typedef if payload_encoding != "none": output_typedef_data["payload_encoding"] = payload_encoding _write_output_typedef_arg(args, output_typedef_data) if args.raw_decode: output = message else: output = { "message": message, "typedef": output_typedef, } if payload_encoding != "none": output["payload_encoding"] = payload_encoding _write_output(args, _to_json(args, output)) return 0 if __name__ == "__main__": sys.exit(main()) blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/000077500000000000000000000000001476504373500217645ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/__init__.py000066400000000000000000000025461476504373500241040ustar00rootroot00000000000000"""The `blackboxprotobuf.lib` module provides APIs for decoding and encoding binary protobuf messages. This module re-exports the functions defined in `blackboxprotobuf.lib.api`, which provides a high level interface for the module and convenience functions. """ # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from .api import * blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/api.py000066400000000000000000001001001476504373500230770ustar00rootroot00000000000000"""The `blackboxprotobuf.lib.api` module provides high level functions for decoding and re-encoding protobuf messages. Most functions take the input data, a type definition and a config object. The 'message_type' or type definition (typedef) is a blackboxprotobuf specific format which defines which types should be used for decoding/encoding each field. It is optional for decoding functions but required for encoding funtions. The decoding function will return a typedef that is require to re-encode the array. If a typedef was provided during decoding, then those types will be used for decoding and the typedef return will be the original typedef + any new fields in the message. The config argument is the Config object from `blackboxprotobuf.lib.config` and allows reconfiguring default types and stores "known" message typedefs that can be referenced within other typedefs. This argument can be left out to use a default shared config object. """ # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import re import six import json import collections import blackboxprotobuf.lib.protofile import blackboxprotobuf.lib.types.length_delim import blackboxprotobuf.lib.types.type_maps from blackboxprotobuf.lib.config import default as default_config from blackboxprotobuf.lib.exceptions import ( TypedefException, EncoderException, DecoderException, ) from blackboxprotobuf.lib.typedef import TypeDef if six.PY3: import typing # Circular imports on Config if we don't check here if typing.TYPE_CHECKING: from typing import Dict, List, Optional from blackboxprotobuf.lib.pytypes import Message, TypeDefDict, FieldDefDict from blackboxprotobuf.lib.config import Config def decode_message(buf, message_type=None, config=None): # type: (bytes, Optional[str | TypeDefDict], Optional[Config]) -> tuple[Message, TypeDefDict] """Decode a protobuf message and return a python dictionary representing the message. Args: buf: Bytes representing an encoded protobuf message message_type: Optional type to use as the base for decoding. Allows for customizing field types or names. Can be a python dictionary or a message type name which maps to the `known_types` dictionary in the config. Defaults to an empty definition '{}'. config: `blackboxprotobuf.lib.config.Config` object which allows customizing default types for wire types and holds the `known_types` array. Defaults to `blackboxprotobuf.lib.config.default` if not provided. Returns: A tuple containing a python dictionary representing the message and a type definition for re-encoding the message. The type definition is based on the `message_type` argument if one was provided, but may add additional fields if new fields were encountered during decoding. """ if config is None: config = default_config if isinstance(buf, bytearray): buf = bytes(buf) buf = six.ensure_binary(buf) if message_type is None: message_type = {} elif isinstance(message_type, six.string_types): if message_type not in config.known_types: message_type = {} else: message_type = config.known_types[message_type] if not isinstance(message_type, dict): raise DecoderException( "Decode message received an invalid typedef type. Typedef should be a string with a message name, a dictionary, or None" ) value, typedef, _, _ = blackboxprotobuf.lib.types.length_delim.decode_message( buf, config, TypeDef.from_dict(message_type) ) return value, typedef.to_dict() def encode_message(value, message_type, config=None): # type: (Message, str | TypeDefDict, Optional[Config]) -> bytes """Re-encode a python dictionary as a binary protobuf message. Args: value: Python dictionary to re-encode to bytes. This should usually be a modified version of the dictionary returned by `decode_message`. message_type: Type definition to use to re-encode the message. This will should generally be the type definition returned from the original `decode_message` call. config: `blackboxprotobuf.lib.config.Config` object which allows customizing default types for wire types and holds the `known_types` array. Defaults to `blackboxprotobuf.lib.config.default` if not provided. Returns: A bytearray containing the encoded protobuf message. """ if config is None: config = default_config if message_type is None: raise EncoderException( "Encode message must have valid type definition. message_type cannot be None" ) if isinstance(message_type, six.string_types): if message_type not in config.known_types: raise EncoderException( "The provided message type name (%s) is not known. Encoding requires a valid type definition" % message_type ) message_type = config.known_types[message_type] if not isinstance(message_type, dict): raise EncoderException( "Encode message received an invalid typedef type. Typedef should be a string with a message name or a dictionary." ) return bytes( blackboxprotobuf.lib.types.length_delim.encode_message( value, config, TypeDef.from_dict(message_type) ) ) def protobuf_to_json(buf, message_type=None, config=None): # type: (bytes | list[bytes], Optional[str | TypeDefDict], Optional[Config]) -> tuple[str, TypeDefDict] """Decode a protobuf messages and return a JSON string representing the messages. Args: buf: One or more bytes representing encoded protobuf messages message_type: Optional type to use as the base for decoding. Allows for customizing field types or names. Can be a python dictionary or a message type name which maps to the `known_types` dictionary in the config. Defaults to an empty definition '{}'. config: `blackboxprotobuf.lib.config.Config` object which allows customizing default types for wire types and holds the `known_types` array. Defaults to `blackboxprotobuf.lib.config.default` if not provided. Returns: A tuple containing a JSON string representing the messages and a type definition for re-encoding the messages. The JSON string and type definition are annotated and sorted for readability. The type definition is based on the `message_type` argument if one was provided, but may add additional fields if new fields were encountered during decoding. """ values = [] bufs = buf if isinstance(buf, list) else [buf] if len(bufs) == 0: raise DecoderException("No protobuf bytes were provided") for data in bufs: value, message_type = decode_message(data, message_type, config) value = _json_safe_transform(value, message_type, False, config=config) value = _sort_output(value, message_type, config=config) values.append(value) if not isinstance(message_type, dict): # Shouldn't happen because of len(bufs) check, but make the type checker happy and verify edge cases raise DecoderException( "Error decoding to json: Could not find valid message_type type (dict). Found: %s" % type(message_type) ) _annotate_typedef(message_type, values[0]) message_type = sort_typedef(message_type) if not isinstance(buf, list) and len(values) == 1: return json.dumps(values[0], indent=2), message_type else: return json.dumps(values, indent=2), message_type def protobuf_from_json(json_str, message_type, config=None): # type: (str, str | TypeDefDict, Optional[Config]) -> bytes | list[bytes] """Re-encode a JSON string as a binary protobuf message. Args: json_str: JSON string to re-encode to protobuf message bytes. This should usually be a modified version of the value returned by `protobuf_to_json`. message_type: Type definition to use to re-encode the message. This will should generally be the type definition returned from the original `protobuf_to_json` call. config: `blackboxprotobuf.lib.config.Config` object which allows customizing default types for wire types and holds the `known_types` array. Defaults to `blackboxprotobuf.lib.config.default` if not provided. Returns: A bytearray containing the encoded protobuf message. """ if config is None: config = default_config if isinstance(message_type, six.string_types): if message_type not in config.known_types: raise EncoderException( 'protobuf_from_json must have valid type definition. message_type "%s" is not known' % message_type ) message_type = config.known_types[message_type] if not isinstance(message_type, dict): raise EncoderException( "Encode message received an invalid typedef type. Typedef should be a string with a message name or a dictionary." ) value = json.loads(json_str) values = value if isinstance(value, list) else [value] _strip_typedef_annotations(message_type) values = [_json_safe_transform(message, message_type, True) for message in values] payloads = [] for message in values: payloads.append(encode_message(message, message_type, config)) if not isinstance(value, list) and len(payloads) == 1: return payloads[0] else: return payloads def export_protofile(message_types, output_filename): # type: (Dict[str, TypeDefDict], str) -> None """This function attempts to export a set of message type definitions to a `.proto` file for use with other tools. Args: message_types: Python dictionary containing the type definitions to export. The dictionary should contain the message type name as the key and the type definition as the value. output_filename: String representing the filename to output the protobuf definition file to. """ blackboxprotobuf.lib.protofile.export_proto( message_types, output_filename=output_filename ) def import_protofile(input_filename, save_to_known=True, config=None): # type: (str, bool, Optional[Config]) -> Dict[str, TypeDefDict] | None """This function attempts to import a set of message type definitions from a `.proto` file. This is a convenience function for `blackboxprotobuf.lib.protofile`. The protobuf file import support is not complete and may fail for some type definitions. Args: input_filename: Filename to read the protobuf definitions from. save_to_known: If True, this function will save the message type definitions to `config.known_types`. Otherwise, it will return them to the caller. Defaults to `True`. config: Optional config object which stores the `known_types` map. Defaults to `blackboxprotobuf.lib.config.default`. Returns: If `save_to_known` is False, then the type definitions read from the file are returned as a dictionary, with the type names as the keys and type definitions as the values. """ if config is None: config = default_config new_typedefs = blackboxprotobuf.lib.protofile.import_proto( config, input_filename=input_filename ) if save_to_known: config.known_types.update(new_typedefs) return None else: return new_typedefs NAME_REGEX = re.compile(r"\A[a-zA-Z][a-zA-Z0-9_]*\Z") def validate_typedef(typedef, old_typedef=None, config=None, _path=None): # type: (TypeDefDict, Optional[TypeDefDict], Optional[Config], Optional[List[str]]) -> None """Attempt to validate a type definition object is valid. This function attempts to ensure a type definition is valid before it is used to encode/decode a message. This will make sure the field names are valid and field names/numbers are consistent. It is intended to be called after a user has edited the type definition to ensure the edits are valid. Args: typedef: The type definition object to validate. This should be a python dict derived from the dict returned by a decode function. old_typedef: Optionally provide a old version of the type definition to compare the new type definnition to. If provided, this function will ensure any type changes are valid. For example, a field with a varint type can be changed to other varint types, but not a string or float. config: Optionally provide a config object which contains the `known_types` map to map message type names to known type definitions. Defaults to `blackboxprotobuf.lib.config.default`. Raises: TypedefException: Raises a TypedefException if the provided type definition is not valid. """ if _path is None: _path = [] if config is None: config = default_config int_keys = set() field_names = set() for field_number, field_typedef in typedef.items(): alt_field_number = None if "-" in str(field_number): field_number, alt_field_number = field_number.split("-") # Validate field_number is a number if not str(field_number).isdigit(): raise TypedefException("Field number must be a digit: %s" % field_number) field_number = six.ensure_text(str(field_number)) field_path = _path[:] field_path.append(field_number) # Check for duplicate field numbers if field_number in int_keys: raise TypedefException( "Duplicate field number: %s" % field_number, field_path ) int_keys.add(field_number) # Must have a type field if "type" not in field_typedef: raise TypedefException( "Field number must have a type value: %s" % field_number, field_path ) if alt_field_number is not None: if field_typedef["type"] != "message": raise TypedefException( "Alt field number (%s) specified for non-message field: %s" % (alt_field_number, field_number), field_path, ) valid_type_fields = [ "type", "name", "field_order", "message_typedef", "message_type_name", "alt_typedefs", "example_value_ignored", "seen_repeated", ] for key, value in field_typedef.items(): # Check field keys against valid values if key not in valid_type_fields: raise TypedefException( 'Invalid field key "%s" for field number %s' % (key, field_number), field_path, ) if ( key in ["message_typedef", "message_type_name"] and not field_typedef["type"] == "message" ): raise TypedefException( 'Invalid field key "%s" for field number %s' % (key, field_number), field_path, ) if key == "group_typedef" and not field_typedef["type"] == "group": raise TypedefException( 'Invalid field key "%s" for field number %s' % (key, field_number), field_path, ) # Validate type value if key == "type": if value not in blackboxprotobuf.lib.types.type_maps.WIRETYPES: raise TypedefException( 'Invalid type "%s" for field number %s' % (value, field_number), field_path, ) # Check for duplicate names if key == "name": if not isinstance(value, six.string_types): raise TypedefException( "Invalid type for name field in typedef: %r. Field number %s" % (value, field_number), field_path, ) if value.strip() == "": continue if value.lower() in field_names: raise TypedefException( ('Duplicate field name "%s" for field ' "number %s") % (value, field_number), field_path, ) if not NAME_REGEX.match(value): raise TypedefException( ( 'Invalid field name "%s" for field ' "number %s. Should match %s" ) % (value, field_number, "[a-zA-Z_][a-zA-Z0-9_]*"), field_path, ) field_names.add(value.lower()) # Check if message type name is known if key == "message_type_name": if value not in config.known_types: raise TypedefException( ( 'Message type "%s" for field number' " %s is not known. Known types: %s" ) % (value, field_number, config.known_types.keys()), field_path, ) # Recursively validate inner typedefs if key == "message_typedef": if isinstance(value, dict): if ( old_typedef is not None and field_number in old_typedef and key in old_typedef[field_number] ): validate_typedef( value, old_typedef=old_typedef[field_number]["message_typedef"], _path=field_path, config=config, ) else: validate_typedef(value, _path=field_path, config=config) if key == "alt_typedefs": for alt_field_number, alt_typedef in field_typedef[ "alt_typedefs" ].items(): if isinstance(alt_typedef, dict): validate_typedef(alt_typedef, _path=field_path, config=config) if old_typedef is not None: wiretype_map = {} for field_number, value in old_typedef.items(): wiretype_map[ int(field_number) ] = blackboxprotobuf.lib.types.type_maps.WIRETYPES[value["type"]] for field_number, value in typedef.items(): field_path = _path[:] field_path.append(str(field_number)) if int(field_number) in wiretype_map: old_wiretype = wiretype_map[int(field_number)] if ( old_wiretype != blackboxprotobuf.lib.types.type_maps.WIRETYPES[value["type"]] ): raise TypedefException( ( "Wiretype for field number %s does" " not match old type definition" ) % field_number, field_path, ) def _json_safe_transform(values, typedef, toBytes, config=None): # type: (Message, TypeDefDict, bool, Optional[Config]) -> Message # Python's JSON doesn't have a default way to handle 'bytes' types. To # handle this, we want some string like encoding which JSON can handle but # can also handle arbitrary bytes. This method get's more complicated than # just converting all bytes since on re-encoding we need to know which ones # were transformed and which are supposed to actually be strings # A built-for binary encoding method like hex or base64 would be 'proper', # but doesn't really give any information to a reader. In some cases, a # binary blob may have embedded strings or integer values that would be # beneficial to quickly skim. # This uses latin1 encoding because it can handle arbitrary bytes, prints # ASCII characters and can be decoded back to the same exact byte string. # It's possible I missed another encoding method that matches these # properties across python2.7 and python3.9, but had issues with some other # backslash escape mechanisms parsing back to bytes. if config is None: config = default_config name_map = { item["name"]: number for number, item in typedef.items() if ("name" in item and item["name"] != "") } if not isinstance(values, dict): # this function should only ever be called on a message, error out if # it is not one. This usually means a type got swapped around raise EncoderException( "Error performing _json_safe_transform on message. Field was expected to be a message but was not: %r" % values ) for name, value in values.items(): if isinstance(name, int): name = six.ensure_text(str(name)) alt_number = None base_name = name if "-" in name: base_name, alt_number = name.split("-") if base_name in name_map: field_number = name_map[base_name] else: field_number = base_name if field_number not in typedef or "type" not in typedef[field_number]: raise EncoderException( "Field %r not found in typedef or does not have type attribute." % field_number ) field_type = typedef[field_number]["type"] # type: str | TypeDefDict if field_type == "message": field_typedef = _get_typedef_for_message(typedef[field_number], config) if alt_number is not None: # if we have an alt type, then let's look that up instead if alt_number not in typedef[field_number].get("alt_typedefs", {}): raise TypedefException( ( "Provided alt field name/number " "%s is not valid for field_number %s" ) % (alt_number, field_number) ) field_type = typedef[field_number]["alt_typedefs"][alt_number] if isinstance(field_type, dict): field_typedef = field_type field_type = "message" is_list = isinstance(value, list) field_values = value if is_list else [value] for i, field_value in enumerate(field_values): if field_type == "bytes": if toBytes: field_values[i] = field_value.encode("latin1") else: field_values[i] = field_value.decode("latin1") elif field_type == "message": field_values[i] = _json_safe_transform( field_value, field_typedef, toBytes, config=config, ) # convert back to single value if needed if not is_list: values[name] = field_values[0] else: values[name] = field_values return values def _get_typedef_for_message(field_typedef, config): # type: (FieldDefDict, Config) -> TypeDefDict assert field_typedef["type"] == "message" if "message_typedef" in field_typedef: return field_typedef["message_typedef"] elif field_typedef.get("message_type_name"): if field_typedef["message_type_name"] not in config.known_types: raise TypedefException( "Got 'message_type_name' not in known_messages: %s" % field_typedef["message_type_name"] ) return config.known_types[field_typedef["message_type_name"]] else: raise TypedefException( "Got 'message' type without typedef or type name: %s" % field_typedef ) def _sort_output(value, typedef, config=None): # type: (Message, TypeDefDict, Optional[Config]) -> Message # Sort output by the field number in the typedef. Helps with readability in # a JSON dump output_dict = collections.OrderedDict() # type: Message if config is None: config = default_config # Make a list of all the field names we have, aggregate together the alt fields as well field_names = {} # type: Dict[str, List[tuple[str, str | None]]] for field_name in value.keys(): if isinstance(field_name, int): field_name = six.ensure_text(str(field_name)) if "-" in field_name: field_name_base, alt_number = field_name.split("-") else: field_name_base = field_name alt_number = None field_names.setdefault(field_name_base, []).append((field_name, alt_number)) for field_number, field_def in sorted(typedef.items(), key=lambda t: int(t[0])): field_number = six.ensure_text(str(field_number)) seen_field_names = field_names.get(field_number, []) # Try getting matching fields by name as well if field_def.get("name", "") != "": field_name = field_def["name"] seen_field_names.extend(field_names.get(field_name, [])) for field_name, alt_number in seen_field_names: field_type = field_def["type"] field_message_typedef = None if field_type == "message": field_message_typedef = _get_typedef_for_message(field_def, config) if alt_number is not None: if alt_number not in field_def["alt_typedefs"]: raise TypedefException( ( "Provided alt field name/number " "%s is not valid for field_number %s" ) % (alt_number, field_number) ) alt_field_type = field_def["alt_typedefs"][alt_number] if isinstance(alt_field_type, dict): field_message_typedef = alt_field_type field_type = "message" else: field_type = alt_field_type if field_type == "message": if field_message_typedef is None: raise TypedefException( 'Message does not have an associated typedef: "%s"' % field_name ) field_value = value.get(field_name) if isinstance(field_value, list): output_dict[field_name] = [] for field_value_item in field_value: if not isinstance(field_value_item, dict): raise TypedefException( 'Message values must be a dictionary type. Field name: "%s"' % field_name ) output_dict[field_name].append( _sort_output(field_value_item, field_message_typedef) ) else: if not isinstance(field_value, dict): raise TypedefException( 'Message values must be a dictionary type. Field name: "%s"' % field_name ) output_dict[field_name] = _sort_output( field_value, field_message_typedef ) else: output_dict[field_name] = value[field_name] return output_dict def sort_typedef(typedef): # type: (TypeDefDict) -> TypeDefDict """Apply special sorting rules to the type definition to improve readability. Sorts the fields of a type definition so that important fields such as the 'type' or 'name' are at the top and don't get buried beneath longer fields like 'message_typedef'. This will also sort the keys of the 'message_typedef' based on the field number. Args: typedef - dictionary representing a Blackboxprotobuf type definition Returns: A new OrderedDict object containing the contents of the typedef argument sorted for readability. """ # Sort output by field number and sub_keys so name then type is first TYPEDEF_KEY_ORDER = [ "name", "type", "message_type_name", "example_value_ignored", "field_order", "seen_repeated", "message_typedef", "alt_typedefs", ] output_dict = collections.OrderedDict() for field_number, field_def in sorted( typedef.items(), key=lambda t: int(t[0]) ): # Sort by type number output_field_def = collections.OrderedDict() for key, value in sorted( field_def.items(), key=lambda t: (TYPEDEF_KEY_ORDER.index(t[0]), t[1]) ): # sort by special keys, then value if key == "message_typedef": output_field_def[key] = sort_typedef(value) # type: ignore else: output_field_def[key] = value # type: ignore output_dict[field_number] = output_field_def if six.PY3 and typing.TYPE_CHECKING: return typing.cast( TypeDefDict, output_dict ) # Cast because typing doesn't like the ordered dict return output_dict def _annotate_typedef(typedef, message): # type: (TypeDefDict, Message) -> None # Add values from message into the typedef so it's easier to figure out # which field when you're editing manually for field_number, field_def in typedef.items(): field_value = None field_name = six.ensure_text(str(field_number)) if field_name not in message and field_def.get("name", "") != "": field_name = field_def["name"] if field_name in message: field_value = message[field_name] if field_def["type"] == "message": if isinstance(field_value, list): for value in field_value: _annotate_typedef(field_def["message_typedef"], value) else: _annotate_typedef(field_def["message_typedef"], field_value) else: field_def["example_value_ignored"] = field_value # Add a blank name field if the field doesn't have one, so it's easier # to add if "name" not in field_def: field_def["name"] = six.u("") def _strip_typedef_annotations(typedef): # type: (TypeDefDict) -> None # Remove example values placed by _annotate_typedef for _, field_def in typedef.items(): if "example_value_ignored" in field_def: del field_def["example_value_ignored"] if "message_typedef" in field_def: _strip_typedef_annotations(field_def["message_typedef"]) blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/config.py000066400000000000000000000053141476504373500236060ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from .types import type_maps import six from blackboxprotobuf.lib.exceptions import ( DecoderException, ) if six.PY3: import typing if typing.TYPE_CHECKING: from typing import Dict from .pytypes import TypeDefDict class Config: def __init__(self): # type: (Config) -> None # Map of message type names to typedefs, previously stored at # `blackboxprotobuf.known_messages` self.known_types = {} # type: Dict[str, TypeDefDict] # Default type for "bytes" like objects that aren't messages or strings # Other option is currently just 'bytes_hex' self.default_binary_type = "bytes" # Change the default type for a wiretype (eg. change ints to be signed # by default or fixed fields to always be float) self.default_types = {} # type: Dict[int, str] # Configure whether bbpb should try to re-encode fields in the same # order they decoded # Field order shouldn't matter for real protobufs, but is there to ensure # that bytes/string are accidentally valid protobufs don't get scrambled # by decoding/re-encoding self.preserve_field_order = True def get_default_type(self, wiretype): # type: (Config, int) -> str default_type = self.default_types.get(wiretype, None) if default_type is None: default_type = type_maps.WIRE_TYPE_DEFAULTS.get(wiretype, None) if default_type is None: raise DecoderException( "Could not find default type for wire type %d" % wiretype ) return default_type default = Config() blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/exceptions.py000066400000000000000000000101601476504373500245150ustar00rootroot00000000000000"""Exception classes for BlackboxProtobuf""" # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import six if six.PY3: from typing import Any, Optional, List class BlackboxProtobufException(Exception): """Base class for excepions raised by Blackbox Protobuf""" def __init__(self, message, path=None, *args): # type: (str, Optional[List[str]], Any) -> None self.path = path super(BlackboxProtobufException, self).__init__(message, *args) def set_path(self, path): # type: (BlackboxProtobufException, List[str]) -> None if self.path is None: self.path = path class TypedefException(BlackboxProtobufException): """Thrown when an error is identified in the type definition, such as conflicting or inconsistent values.""" def __str__(self): # type: (TypedefException) -> str message = super(TypedefException, self).__str__() if self.path is not None: message = ( "Encountered error within typedef for field %s: " % "->".join(map(str, self.path)) ) + message else: message = ("Encountered error within typedef: ") + message return message class EncoderException(BlackboxProtobufException, ValueError): """Thrown when there is an error encoding a dictionary to a type definition""" def __str__(self): # type: (EncoderException) -> str message = super(EncoderException, self).__str__() if self.path is not None: message = ( "Encountered error encoding field %s: " % "->".join(map(str, self.path)) ) + message else: message = ("Encountered error encoding message: ") + message return message class DecoderException(BlackboxProtobufException, ValueError): """Thrown when there is an error decoding a bytestring to a dictionary""" def __str__(self): # type: (DecoderException) -> str message = super(DecoderException, self).__str__() if self.path is not None: message = ( "Encountered error decoding field %s: " % "->".join(map(str, self.path)) ) + message else: message = ("Encountered error decoding message: ") + message return message class ProtofileException(BlackboxProtobufException): def __init__(self, message, path=None, filename=None, *args): # type: (ProtofileException, str, Optional[List[str]], Optional[str], Any) -> None self.filename = filename super(BlackboxProtobufException, self).__init__(message, path, *args) def __str__(self): # type: (ProtofileException) -> str message = super(ProtofileException, self).__str__() if self.path is not None: message = ( "Encountered error within protofile %s for field %s: " % (self.filename, "->".join(map(str, self.path))) ) + message else: message = ( "Encountered error within protofile %s: " % self.filename ) + message return message blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/payloads/000077500000000000000000000000001476504373500236005ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/payloads/__init__.py000066400000000000000000000065401476504373500257160ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ The payloads module is intended to handle different encodings of the protobuf data, such as compression and grpc header. """ from blackboxprotobuf.lib.exceptions import BlackboxProtobufException from . import gzip, grpc import six if six.PY3: from typing import List, Callable, Tuple, Optional # Returns an ordered list of potential decoders, from most specific to least specific # The consumer should loop through each decoder, try to decode it and then try # to decode as a protobuf. This should minimize the chance of a false positive # on any decoders def find_decoders(buf): # type: (bytes) -> List[Callable[[bytes], Tuple[bytes | list[bytes], str]]] # In the future, we can take into account content-type too, such as for # grpc, but we risk false negatives decoders = [] # type: List[Callable[[bytes], Tuple[bytes | list[bytes], str]]] if gzip.is_gzip(buf): decoders.append(gzip.decode_gzip) if grpc.is_grpc(buf): decoders.append(grpc.decode_grpc) decoders.append(_none_decoder) return decoders def _none_decoder(buf): # type: (bytes) -> Tuple[bytes, str] return buf, "none" # Decoder by name def decode_payload(buf, decoder): # type: (bytes, Optional[str]) -> Tuple[bytes | list[bytes], str] if decoder is None: return buf, "none" decoder = decoder.lower() if decoder == "none": return buf, "none" elif decoder.startswith("grpc"): return grpc.decode_grpc(buf) elif decoder == "gzip": return gzip.decode_gzip(buf) else: raise BlackboxProtobufException("Unknown decoder: " + decoder) # Encode by name, should pass in the results from the decode function def encode_payload(buf, encoder): # type: (bytes | list[bytes], Optional[str]) -> bytes if encoder is None: encoder = "none" encoder = encoder.lower() if encoder == "none": if isinstance(buf, list): raise BlackboxProtobufException( "Cannot encode multiple buffers with none/missing encoding" ) return buf elif encoder.startswith("grpc"): return grpc.encode_grpc(buf, encoder) elif encoder == "gzip": return gzip.encode_gzip(buf) else: raise BlackboxProtobufException("Unknown encoder: " + encoder) blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/payloads/grpc.py000066400000000000000000000102361476504373500251070ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import six import struct from blackboxprotobuf.lib.exceptions import BlackboxProtobufException if six.PY3: from typing import Tuple # gRPC over HTTP2 spec: https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md HEADER_LEN = 1 + 4 def is_grpc(payload): # type: (bytes) -> bool if len(payload) < HEADER_LEN: return False if six.PY2 and isinstance(payload, bytearray): payload = bytes(payload) pos = 0 while pos < len(payload): compression_byte = six.indexbytes(payload, pos) # Change this to support 0x1 once we support compression if compression_byte != 0: return False message_length = struct.unpack_from(">I", payload[pos + 1 : pos + 5])[0] pos += message_length + 5 if pos != len(payload): return False return True def decode_grpc(payload): # type: (bytes) -> Tuple[bytes | list[bytes], str] """Decode GRPC. Return the protobuf data""" if six.PY2 and isinstance(payload, bytearray): payload = bytes(payload) if len(payload) == 0: raise BlackboxProtobufException("Error decoding GRPC. Payload is empty") pos = 0 payloads = [] while pos + HEADER_LEN <= len(payload): compression_byte = six.indexbytes(payload, pos) pos += 1 if compression_byte != 0x00: if compression_byte == 0x01: # Payload is compressed # If a payload is compressed, the compression method is specified in the `grpc-encoding` header # Options are "identity" / "gzip" / "deflate" / "snappy" / {custom} raise BlackboxProtobufException( "Error decoding GRPC. Compressed payloads are not supported" ) else: raise BlackboxProtobufException( "Error decoding GRPC. First byte must be 0 or 1 to indicate compression" ) message_length = struct.unpack_from(">I", payload[pos : pos + 4])[0] pos += 4 if len(payload) < pos + message_length: raise BlackboxProtobufException( "Error decoding GRPC. Payload length does not match encoded gRPC length" ) payloads.append(payload[pos : pos + message_length]) pos += message_length if pos != len(payload): raise BlackboxProtobufException( "Error decoding GRPC. Payload length does not match encoded gRPC lengths" ) if len(payloads) > 1: return payloads, "grpc" else: return payloads[0], "grpc" def encode_grpc(data, encoding="grpc"): # type: (bytes | list[bytes], str) -> bytes if encoding != "grpc": raise BlackboxProtobufException( "Error encoding GRPC with encoding %s. GRPC is only supported with no compression" % encoding ) datas = data if isinstance(data, list) else [data] payload = bytearray() for data in datas: payload.append(0x00) # No compression payload.extend(struct.pack(">I", len(data))) # Length payload.extend(data) return payload blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/payloads/gzip.py000066400000000000000000000037351476504373500251330ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import zlib from blackboxprotobuf.lib.exceptions import BlackboxProtobufException import six if six.PY3: from typing import Tuple def is_gzip(buf): # type: (bytes) -> bool return buf.startswith(bytearray([0x1F, 0x8B, 0x08])) def decode_gzip(buf): # type: (bytes) -> Tuple[bytes, str] if buf.startswith(bytearray([0x1F, 0x8B, 0x08])): decompressor = zlib.decompressobj(31) return decompressor.decompress(buf), "gzip" else: raise BlackboxProtobufException( "Cannot decode as gzip: magic bytes don't match" ) def encode_gzip(buf): # type: (bytes | list[bytes]) -> bytes if isinstance(buf, list): raise BlackboxProtobufException( "Cannot encode as gzip: multiple buffers are not supported" ) compressor = zlib.compressobj(-1, zlib.DEFLATED, 31) return compressor.compress(buf) + compressor.flush() blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/protofile.py000066400000000000000000000423721476504373500243510ustar00rootroot00000000000000""" Python methods for importing and exporting '.proto' files from the BBP type definition format. """ # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import io import re import logging import re import six from blackboxprotobuf.lib.exceptions import TypedefException, ProtofileException import blackboxprotobuf.lib.api if six.PY3: import typing if typing.TYPE_CHECKING: from blackboxprotobuf.lib.config import Config from typing import Any, TextIO, Tuple, Dict, Optional, List from blackboxprotobuf.lib.pytypes import TypeDefDict, FieldDefDict logger = logging.getLogger(__name__) PROTO_FILE_TYPE_MAP = { "uint": "uint64", "int": "int64", "sint": "sint64", "fixed32": "fixed32", "sfixed32": "sfixed32", "float": "float", "fixed64": "fixed64", "sfixed64": "sfixed64", "double": "double", "bytes": "bytes", "bytes_hex": "bytes", "string": "string", } PACKABLE_TYPES = [ "uint", "int", "sint", "fixed32", "sfixed32", "float", "fixed64", "sfixed64", "double", ] # Inverse of the above, but we have to include more types PROTO_FILE_TYPE_TO_BBP = { "double": "double", "float": "float", "int32": "int", "int64": "int", "uint32": "uint", "uint64": "uint", "sint32": "sint", "sint64": "sint", "fixed32": "fixed32", "fixed64": "fixed64", "sfixed32": "sfixed32", "sfixed64": "sfixed64", "bool": "uint", "string": "string", # should be default_binary_type, but can't handle that well here "bytes": "bytes", } NAME_REGEX = re.compile(r"\A[a-zA-Z_][a-zA-Z0-9_]*\Z") # add packed types to the list for packable_type in PACKABLE_TYPES: packed_type = "packed_" + packable_type PROTO_FILE_TYPE_MAP[packed_type] = PROTO_FILE_TYPE_MAP[packable_type] def _print_message(message_name, typedef, output_file, depth=0): # type: (str, TypeDefDict, TextIO, int) -> None indent = six.u(" ") * depth if not NAME_REGEX.match(message_name): raise TypedefException("Message name: %s is not valid" % message_name) # sort typedef for better looking output typedef = blackboxprotobuf.lib.api.sort_typedef(typedef) message_name = message_name.strip() output_file.write(six.u("\n")) output_file.write(indent) output_file.write(six.u("message %s {\n") % message_name) for field_number, field_typedef in typedef.items(): proto_type = None field_name = None field_options = "" # a repeated field with one element is indistinduishable from a # repeated field so we just put repeated if we have proof that it is # repeatable, but this might be wrong sometimes # maybe some sort of protobuf discovery tool can detect this is_repeated = field_typedef.get("seen_repeated", False) if "name" in field_typedef and field_typedef["name"] != "": field_name = field_typedef["name"] field_name = field_name.strip() if not NAME_REGEX.match(field_name): field_name = None if field_name is None: field_name = six.ensure_text("field%s" % field_number) if field_typedef["type"] == "message": # If we have multiple typedefs, this means is something like the Any # message, and has to be manually reparsed to each type if "alt_typedefs" in field_typedef: proto_type = "bytes" else: proto_type = field_name + "_type" _print_message( proto_type, field_typedef["message_typedef"], output_file, depth + 1 ) else: if field_typedef["type"] not in PROTO_FILE_TYPE_MAP: raise TypedefException( "Type %s does not have a mapping to protobuf types." % field_typedef["type"] ) proto_type = PROTO_FILE_TYPE_MAP[field_typedef["type"]] # we're using proto3 syntax. Repeated numeric fields are packed by default # if it's repeated and not packed, then make sure we specify it's not packed if is_repeated and field_typedef["type"] in PACKABLE_TYPES: field_options = " [packed=false]" # if it's a packed type, we'll explicitoly set that too, can't hurt elif field_typedef["type"].startswith("packed_"): field_options = " [packed=true]" is_repeated = True output_file.write(indent) output_file.write( six.u(" %s%s %s = %s%s;\n") % ( "repeated " if is_repeated else "", proto_type, field_name, field_number, field_options, ) ) output_file.write(indent) output_file.write(six.u("}\n\n")) def export_proto(typedef_map, output_filename=None, output_file=None, package=None): # type: (Dict[str, TypeDefDict], Optional[str], Optional[TextIO], Optional[str]) -> str | None """Export the given type definitons as a '.proto' file. Typedefs are expected as a dictionary of {'message_name': typedef } Write to output_file or output_filename if provided, otherwise return a string output_filename will be overwritten if it exists """ if output_filename is not None: output_file = io.open(output_filename, "w+") if output_file is None: output_file = io.StringIO() # preamble output_file.write(six.u('syntax = "proto3";\n\n')) if package: output_file.write(six.u("package %s;\n\n") % package) for typedef_name, typedef in typedef_map.items(): _print_message(typedef_name, typedef, output_file) if isinstance(output_file, io.StringIO): return output_file.getvalue() # close the file if we opened it elif output_filename is not None: output_file.close() return None MESSAGE_START_REGEX = re.compile(r"^message +([a-zA-Z_0-9]+) *{.*") FIELD_REGEX = re.compile( r"^ *(repeated|optional|required)? *([a-zA-Z0-9_]+) +([a-zA-Z0-9_]+) += +([0-9]+) *(\[[a-z]+=[a-z]*\])?.*;.*$" ) SYNTAX_REGEX = re.compile(r'^ *syntax += +"(proto\d)" *;.*') ENUM_REGEX = re.compile(r"^ *enum +([a-zA-Z0-9_]+) *{.*") PACKAGE_REGEX = re.compile(r"^ *package +([a-zA-Z0-9_.]+) *;.*") def import_proto(config, input_string=None, input_filename=None, input_file=None): # type: (Config, Optional[str], Optional[str], Optional[TextIO]) -> Dict[str, TypeDefDict] typedef_map = {} # type: Dict[str, TypeDefDict] if input_string is not None: input_file = io.StringIO(input_string) if input_file is None and input_filename is not None: input_file = io.open(input_filename, "r") if input_file is None: raise ProtofileException( "No file provided to import_proto", filename=input_filename ) syntax_version = "proto2" package_prefix = "" enum_names = [] message_trees = [] message_names = [] line = input_file.readline() while line: line = line.strip() if line.startswith("syntax"): syntax_match = SYNTAX_REGEX.match(line) if syntax_match: syntax_version = syntax_match.group(1) elif line.startswith("package"): package_match = PACKAGE_REGEX.match(line) if package_match: package_prefix = package_match.group(1) + "." elif line.startswith("import"): logger.warning( "Proto file has import which is not supported " "by the parser. Ensure the imported files are " "processed first: %s", line, ) elif line.startswith("enum"): enum_match = ENUM_REGEX.match(line) if enum_match: enum_name = _parse_enum(enum_match, line, input_file) enum_names.append(enum_name) elif line.startswith("message"): message_start_match = MESSAGE_START_REGEX.match(line) if message_start_match: message_tree = _preparse_message(message_start_match, line, input_file) message_trees.append(message_tree) line = input_file.readline() for tree in message_trees: new_message_names, new_enum_names = _collect_names(package_prefix, tree) enum_names += new_enum_names message_names += new_message_names logger.debug("Got the following enum_names: %s", enum_names) logger.debug("Got the following message_names: %s", message_names) for tree in message_trees: _parse_message( tree, typedef_map, message_names, enum_names, package_prefix, syntax_version == "proto3", config, ) return typedef_map def _parse_enum(enum_match, line, input_file): # type: (re.Match[str], str, TextIO) -> str """Parse an enum out of the file. Goes from enum declaration to next } Returns the enum's name """ enum_name = enum_match.group(1) # parse until the next '}' while "}" not in line: line = input_file.readline() if not line: raise ProtofileException("Did not find close of enum") return enum_name def _preparse_message(message_start_match, line, input_file): # type: (re.Match[str], str, TextIO) -> Dict[str, Any] # TODO Should put together better types than Any, but we'll stick with this # for now """Parse out a message name and the lines that make it up""" message_name = message_start_match.group(1) message_lines = [] inner_enums = [] inner_messages = [] while "}" not in line: line = input_file.readline() if not line: raise ProtofileException("Did not find close of message") line = line.strip() if line.startswith("enum"): enum_match = ENUM_REGEX.match(line) if enum_match: enum_name = _parse_enum(enum_match, line, input_file) inner_enums.append(enum_name) elif line.startswith("message"): inner_message_start_match = MESSAGE_START_REGEX.match(line) if inner_message_start_match: message_tree = _preparse_message( inner_message_start_match, line, input_file ) inner_messages.append(message_tree) # not an inner enum or message else: message_lines.append(line) return { "name": message_name, "data": message_lines, "enums": inner_enums, "inner_messages": inner_messages, } def _collect_names(prefix, message_tree): # type: (str, Dict[str, Any]) -> Tuple[List[str], List[str]] message_names = [] enum_names = [] name = prefix + message_tree["name"] message_names.append(name) for enum_name in message_tree["enums"]: enum_names.append(prefix + enum_name) for inner_message in message_tree["inner_messages"]: new_message_names, new_enum_names = _collect_names(name + ".", inner_message) message_names += new_message_names enum_names += new_enum_names return message_names, enum_names def _check_message_name(current_path, name, known_message_names, config): # type: (str, str, List[str], Config) -> str | None # Verify message name against preparsed message names and global # known_messages # For example, if we have: # Message.InnerMesage # referenced from: # PackageA.Message2 # we would look up: # PackageA.Message2.Message.InnerMessage # PackageA.Message.InnerMessage # should also work for enums if name in config.known_types: return name # search for anything under a common prefix in known_message_names logger.debug("Testing message name: %s", name) prefix_options = [""] for part in current_path.split("."): if part: prefix_options = [prefix_options[0] + part + "."] + prefix_options logger.debug("prefix_options: %s", prefix_options) for prefix in prefix_options: logger.debug("Testing message name: %s", prefix + name) if prefix + name in known_message_names: return prefix + name # remove the last bit of the prefix if "." not in prefix: break prefix = ".".join(prefix.split(".")[:-1]) logger.debug( "Message %s not found from %s Known names are: %s", name, current_path, known_message_names, ) return None def _parse_message( message_tree, typdef_map, known_message_names, enum_names, prefix, is_proto3, config ): # type: (Dict[str, Any], Dict[str, TypeDefDict], List[str], List[str], str, bool, Config) -> None message_typedef = {} message_name = prefix + message_tree["name"] prefix = message_name + "." # parse the actual message fields for line in message_tree["data"]: # lines should already be stripped and should not have messages or enums assert all([not line.strip().startswith(x) for x in ["message ", "enum "]]) # Check if the line matches the field regex match = FIELD_REGEX.match(line) if match: field_number, field_typedef = _parse_field( match, known_message_names, enum_names, prefix, is_proto3, config ) message_typedef[field_number] = field_typedef # add the messsage to tyep returned typedefs logger.debug("Adding message %s to typedef maps", message_name) typdef_map[message_name] = message_typedef for inner_message in message_tree["inner_messages"]: _parse_message( inner_message, typdef_map, known_message_names, enum_names, prefix, is_proto3, config, ) # parse a field into a dictionary for the typedef def _parse_field(match, known_message_names, enum_names, prefix, is_proto3, config): # type: (re.Match[str], List[str], List[str], str, bool, Config) -> Tuple[str, FieldDefDict] typedef = {} # type: FieldDefDict field_name = match.group(3) if not field_name: raise ProtofileException("Could not parse field name from line: %s" % match) typedef["name"] = six.ensure_text(field_name) field_number = match.group(4) if not field_number: raise ProtofileException("Could not parse field number from line: %s" % match) # figure out repeated field_rule = match.group(1) is_repeated = False if field_rule and "repeated" in field_rule: is_repeated = True typedef["seen_repeated"] = True field_type = match.group(2) if not field_type: raise ProtofileException("Could not parse field type from line: %s" % match) # check normal types bbp_type = PROTO_FILE_TYPE_TO_BBP.get(field_type, None) if not bbp_type: logger.debug("Got non-basic type: %s, checking enums", field_type) # check enum names if _check_message_name(prefix, field_type, enum_names, config): # enum = uint bbp_type = "uint" if not bbp_type: # Not enum or normal type, check messages message_name = _check_message_name( prefix, field_type, known_message_names, config ) if message_name: bbp_type = "message" typedef["message_type_name"] = message_name if not bbp_type: # If we don't have a type now, then fail raise ProtofileException( "Could not get a type for field %s: %s" % (field_name, field_type) ) # figure out packed # default based on repeated + proto3, fallback to options field_options = match.group(5) is_packed = is_repeated and is_proto3 and (bbp_type in PACKABLE_TYPES) if is_packed and field_options and "packed=false" in field_options: is_packed = False elif is_repeated and field_options and "packed=true" in field_options: is_packed = True # make sure the type lines up with packable if is_packed and bbp_type not in PACKABLE_TYPES: raise ProtofileException( "Field %s set as packable, but not a packable type: %s" % (field_name, bbp_type) ) if is_packed: bbp_type = "packed_" + bbp_type typedef["type"] = bbp_type logger.debug("Parsed field number %s: %s", field_number, typedef) return field_number, typedef blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/pytypes.py000066400000000000000000000036171476504373500240620ustar00rootroot00000000000000"""This module provides top level types for adding type definitions to the blackboxprotobuf library. """ # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import six if six.PY3: from typing import Any, Dict, List, TypedDict # We say messages can have any value # Functions we define may have fixed types, but someone could add a type # function that outputs any arbitrary object Message = Dict[str | int, Any] TypeDefDict = Dict[str, "FieldDefDict"] FieldDefDict = TypedDict( "FieldDefDict", { "name": str, "type": str, "message_type_name": str, "message_typedef": TypeDefDict, "alt_typedefs": Dict[str, str | TypeDefDict], "example_value_ignored": Any, "seen_repeated": bool, "field_order": List[str], }, total=False, ) blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/typedef.py000066400000000000000000000247421476504373500240070ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import six from blackboxprotobuf.lib.exceptions import ( BlackboxProtobufException, EncoderException, TypedefException, ) if six.PY3: import typing if typing.TYPE_CHECKING: from typing import Dict, Optional, Any, List, TypedDict, Tuple from blackboxprotobuf.lib.config import Config from .pytypes import TypeDefDict, FieldDefDict class TypeDef(object): def __init__(self): # type: (TypeDef) -> None self._fields = {} # type: Dict[str, FieldDef] self._field_names = {} # type: Dict[str, str] @staticmethod def from_dict(typedef_dict): # type: (TypeDefDict) -> TypeDef typedef = TypeDef() for field_id, fielddef_dict in typedef_dict.items(): fielddef = FieldDef.from_dict(fielddef_dict, field_id) typedef._fields[field_id] = fielddef if fielddef._name: typedef._field_names[fielddef._name] = field_id return typedef def to_dict(self): # type: (TypeDef) -> TypeDefDict return {field: fielddef.to_dict() for field, fielddef in self._fields.items()} def make_mutable(self): # type: (TypeDef) -> MutableTypeDef mutable = MutableTypeDef() # Copy fields but don't deep copy # They should get immutable copies of FieldDef mutable._fields = self._fields.copy() return mutable def lookup_fielddef(self, key): # type: (TypeDef, str) -> Optional[Tuple[str, FieldDef]] """Look up a field definition by number (w/ alt typedef) or name""" # We don't care what the alt number is now field_name = key.split("-", 1)[0] field_id = self._field_names.get(field_name, field_name) if field_id in self._fields: return field_id, self._fields[field_id] return None def lookup_fielddef_number(self, field_id): # type: (TypeDef, str) -> Optional[Tuple[str, FieldDef]] if field_id in self._fields: return field_id, self._fields[field_id] return None class MutableTypeDef(TypeDef): def set_fielddef(self, field_number, fielddef): # type: (MutableTypeDef, str, FieldDef) -> None self._fields[field_number] = fielddef if fielddef._name: self._field_names[fielddef._name] = field_number class FieldDef(object): def __init__(self, field_id): # type: (FieldDef, str) -> None self._name = None # type: Optional[str] self._field_id = field_id # type: str self._message_type_name = None # type: Optional[str] # Normal type will be 0, alts starting at 1 # Each field will have either the type or a typedef # We don't allow message_type_name in alt_typedefs, we'll use it for any "message" entry instead self._types = {} # type: Dict[str, str | TypeDef] self._example_value = None # type: Any self._seen_repeated = False # type: bool self._field_order = None # type: Optional[List[str]] @staticmethod def from_dict(fielddef_dict, field_id): # type: (FieldDefDict, str) -> FieldDef fielddef = FieldDef(field_id) if "name" in fielddef_dict: fielddef._name = fielddef_dict["name"] if "type" in fielddef_dict: field_type = fielddef_dict["type"] if field_type == "message" and "message_typedef" in fielddef_dict: fielddef._types["0"] = TypeDef.from_dict( fielddef_dict["message_typedef"] ) else: fielddef._types["0"] = field_type if "message_type_name" in fielddef_dict: # We could lookup the typedef, but better to wait to resolve fielddef._message_type_name = fielddef_dict["message_type_name"] if "alt_typedefs" in fielddef_dict: for alt_number, alt_typedef in fielddef_dict["alt_typedefs"].items(): if isinstance(alt_typedef, dict): fielddef._types[alt_number] = TypeDef.from_dict(alt_typedef) else: fielddef._types[alt_number] = alt_typedef if "example_value_ignored" in fielddef_dict: fielddef._example_value = fielddef_dict["example_value_ignored"] if "seen_repeated" in fielddef_dict: fielddef._seen_repeated = fielddef_dict["seen_repeated"] if "field_order" in fielddef_dict: fielddef._field_order = fielddef_dict["field_order"] return fielddef def to_dict(self): # type: (FieldDef) -> FieldDefDict fielddef_dict = {} # type: FieldDefDict if self._name: fielddef_dict["name"] = self._name if self._message_type_name: fielddef_dict["message_type_name"] = self._message_type_name if self._example_value: fielddef_dict["example_value_ignored"] = self._example_value if self._seen_repeated: fielddef_dict["seen_repeated"] = self._seen_repeated if self._field_order: fielddef_dict["field_order"] = self._field_order field_type = self._types.get("0") if isinstance(field_type, TypeDef): field_typedef = field_type.to_dict() field_type = "message" fielddef_dict["message_typedef"] = field_typedef if field_type: fielddef_dict["type"] = field_type if field_type and len(self._types) > 1: fielddef_dict["alt_typedefs"] = { alt_num: ( alt_type.to_dict() if isinstance(alt_type, TypeDef) else alt_type ) for alt_num, alt_type in self._types.items() if alt_num != "0" } return fielddef_dict def make_mutable(self): # type: (FieldDef) -> MutableFieldDef mutable = MutableFieldDef(self._field_id) mutable._name = self._name mutable._message_type_name = self._message_type_name mutable._types = self._types.copy() mutable._example_value = self._example_value mutable._seen_repeated = self._seen_repeated mutable._field_order = self._field_order return mutable def lookup_field_type(self, key, config, field_path): # type: (FieldDef, str, Config, List[str]) -> Optional[str | TypeDef] if "-" in key: alt_type_id = key.split("-", 1)[1] else: alt_type_id = "0" return self.lookup_field_type_number(alt_type_id, config, field_path) # Lookup fieled type by just alt type number def lookup_field_type_number(self, alt_type_id, config, field_path): # type: (FieldDef, str, Config, List[str]) -> Optional[str | TypeDef] if alt_type_id not in self._types: return None field_type = self._types[alt_type_id] if field_type == "message": # We have to look up the message type name return self.resolve_message_type_name(config, field_path) return field_type @property def name(self): # type: (FieldDef) -> str if self._name: return self._name return self._field_id def field_key(self, alt_field_id): # type: (FieldDef, str) -> str if alt_field_id == "0": return self.name else: return self.name + "-" + alt_field_id def next_alt_type_id(self): # type: (FieldDef) -> str existing_ids = [int(alt_type_id) for alt_type_id in self._types.keys()] if len(existing_ids) == 0: return "0" else: return six.ensure_text(str(max(existing_ids) + 1)) @property def field_order(self): # type: (FieldDef) -> Optional[List[str]] # someone could mutate the list returned, but that's not a high # priority return self._field_order @property def seen_repeated(self): # type: (FieldDef) -> bool return self._seen_repeated def resolve_message_type_name(self, config, field_path): # type: (FieldDef, Config, List[str]) -> TypeDef if self._message_type_name not in config.known_types: raise TypedefException( "Message name '%s' has not been defined in known types" % self._message_type_name, field_path, ) return TypeDef.from_dict(config.known_types[self._message_type_name]) def resolve_types(self, config, field_path): # type: (FieldDef, Config, List[str]) -> Dict[str, str | TypeDef] field_types = self._types.copy() if field_types.get("0") == "message": field_types["0"] = self.resolve_message_type_name(config, field_path) return field_types class MutableFieldDef(FieldDef): def set_field_order(self, field_order): # type: (MutableFieldDef, List[str]) -> None self._field_order = field_order def mark_repeated(self): # type: (MutableFieldDef) -> None self._seen_repeated = True def set_type(self, alt_type_id, field_type): # type: (MutableFieldDef, str, str | TypeDef) -> None self._types[alt_type_id] = field_type def set_types(self, types): # type: (MutableFieldDef, Dict[str, str | TypeDef]) -> None self._types = types def add_type(self, field_type): # type: (MutableFieldDef, str | TypeDef) -> str alt_type_id = self.next_alt_type_id() self.set_type(alt_type_id, field_type) return alt_type_id blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/types/000077500000000000000000000000001476504373500231305ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/types/__init__.py000066400000000000000000000024201476504373500252370ustar00rootroot00000000000000"""This module contains classes/methods for decoding individual field types. Grouped into submodules based on wire type. Re-exports type_maps items for easier access. """ # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from .type_maps import * blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/types/fixed.py000066400000000000000000000111301476504373500245750ustar00rootroot00000000000000"""Functions for encoding and decoding fixed size integers and floats""" # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import struct import binascii import six from blackboxprotobuf.lib.exceptions import DecoderException, EncoderException import six if six.PY3: from typing import Any, Tuple # Generic functions for encoding/decoding structs based on the "struct" format def encode_struct(fmt, value): # type: (str, Any) -> bytes """Generic method for encoding arbitrary python "struct" values""" try: return struct.pack(fmt, value) except struct.error as exc: six.raise_from( EncoderException( "Error encoding value %r with format string %s" % (value, fmt) ), exc, ) def decode_struct(fmt, buf, pos): # type: (str, bytes, int) -> Tuple[Any, int] """Generic method for decoding arbitrary python "struct" values""" new_pos = pos + struct.calcsize(fmt) try: return struct.unpack(fmt, buf[pos:new_pos])[0], new_pos except struct.error as exc: six.raise_from( DecoderException( "Error deocding format string %s from bytes: %r" % (fmt, binascii.hexlify(buf[pos:new_pos])) ), exc, ) _fixed32_fmt = " bytes """Encode a single 32 bit fixed-size value""" return encode_struct(_fixed32_fmt, value) def decode_fixed32(buf, pos): # type: (bytes, int) -> Tuple[Any, int] """Decode a single 32 bit fixed-size value""" return decode_struct(_fixed32_fmt, buf, pos) _sfixed32_fmt = " bytes """Encode a single signed 32 bit fixed-size value""" return encode_struct(_sfixed32_fmt, value) def decode_sfixed32(buf, pos): # type: (bytes, int) -> Tuple[Any, int] """Decode a single signed 32 bit fixed-size value""" return decode_struct(_sfixed32_fmt, buf, pos) _float_fmt = " bytes """Encode a single 32 bit floating point value""" return encode_struct(_float_fmt, value) def decode_float(buf, pos): # type: (bytes, int) -> Tuple[Any, int] """Decode a single 32 bit floating point value""" return decode_struct(_float_fmt, buf, pos) _fixed64_fmt = " bytes """Encode a single 64 bit fixed-size value""" return encode_struct(_fixed64_fmt, value) def decode_fixed64(buf, pos): # type: (bytes, int) -> Tuple[Any, int] """Decode a single 64 bit fixed-size value""" return decode_struct(_fixed64_fmt, buf, pos) _sfixed64_fmt = " bytes """Encode a single signed 64 bit fixed-size value""" return encode_struct(_sfixed64_fmt, value) def decode_sfixed64(buf, pos): # type: (bytes, int) -> Tuple[Any, int] """Decode a single signed 64 bit fixed-size value""" return decode_struct(_sfixed64_fmt, buf, pos) _double_fmt = " bytes """Encode a single 64 bit floating point value""" return encode_struct(_double_fmt, value) def decode_double(buf, pos): # type: (bytes, int) -> Tuple[Any, int] """Decode a single 64 bit floating point value""" return decode_struct(_double_fmt, buf, pos) blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/types/length_delim.py000066400000000000000000000633111476504373500261410ustar00rootroot00000000000000"""Module for encoding and decoding length delimited fields""" # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import binascii import copy import sys import six import logging import blackboxprotobuf.lib from blackboxprotobuf.lib.types import varint, wiretypes from blackboxprotobuf.lib.exceptions import ( EncoderException, DecoderException, TypedefException, BlackboxProtobufException, ) from blackboxprotobuf.lib.typedef import ( TypeDef, MutableTypeDef, FieldDef, MutableFieldDef, ) if six.PY3: import typing if typing.TYPE_CHECKING: from blackboxprotobuf.lib.config import Config from typing import Any, Callable, Dict, Tuple, Optional, List from blackboxprotobuf.lib.pytypes import Message logger = logging.getLogger(__name__) def encode_string(value): # type: (Any) -> bytes """Encode a string as a length delimited byte array""" try: value = six.ensure_text(value) except TypeError as exc: six.raise_from( EncoderException("Error encoding string to message: %r" % value), exc ) return encode_bytes(value) def encode_bytes(value): # type: (Any) -> bytes """Encode a length delimited byte array""" if isinstance(value, bytearray): value = bytes(value) try: value = six.ensure_binary(value) except TypeError as exc: six.raise_from( EncoderException("Error encoding bytes to message: %r" % value), exc ) if not isinstance(value, bytes): raise EncoderException( "encode_bytes must receive a bytes or bytearray value: %s %r" % (type(value), value) ) encoded_length = varint.encode_varint(len(value)) return encoded_length + value def decode_bytes(buf, pos): # type: (bytes, int) -> Tuple[bytes, int] """Decode a length delimited bytes array from buf""" length, pos = varint.decode_varint(buf, pos) end = pos + length try: return buf[pos:end], end except IndexError as exc: six.raise_from( DecoderException( ( "Error decoding bytes. Decoded length %d is longer than bytes" " available %d" ) % (length, len(buf) - pos) ), exc, ) def encode_bytes_hex(value): # type: (Any) -> bytes """Encode a length delimited byte array represented by a hex string""" try: return encode_bytes(binascii.unhexlify(value)) except (TypeError, binascii.Error) as exc: six.raise_from( EncoderException("Error encoding hex bytestring %s" % value), exc ) def decode_bytes_hex(buf, pos): # type: (bytes, int) -> Tuple[bytes, int] """Decode a length delimited byte array from buf and return a hex encoded string""" value, pos = decode_bytes(buf, pos) return binascii.hexlify(value), pos def decode_string(value, pos): # type: (bytes, int) -> Tuple[str, int] """Decode a length delimited byte array as a string""" length, pos = varint.decode_varint(value, pos) end = pos + length try: # backslash escaping isn't reversible easily return value[pos:end].decode("utf-8"), end except (TypeError, UnicodeDecodeError) as exc: six.raise_from( DecoderException("Error decoding UTF-8 string %r" % value[pos:end]), exc ) def encode_tag(field_number, wire_type): # type: (int, int) -> bytes # Not checking bounds here, should be check before tag_number = (field_number << 3) | wire_type return varint.encode_uvarint(tag_number) def decode_tag(buf, pos): # type: (bytes, int) -> Tuple[int, int, int] tag_number, pos = varint.decode_uvarint(buf, pos) field_number = tag_number >> 3 wire_type = tag_number & 0x7 return field_number, wire_type, pos def encode_message(data, config, typedef, path=None, field_order=None): # type: (Message, Config, TypeDef, Optional[List[str]], Optional[List[str]]) -> bytes """Encode a Python dictionary to a binary protobuf message""" output = bytearray() if path is None: path = [] output_len = 0 field_outputs = {} # type: Dict[str, List[bytes]] for field_id, value in data.items(): field_number, outputs = _encode_message_field( config, typedef, path, field_id, value ) # In case the field number is represented in multiple locations in data # (eg. as an int, as name, as a string with an int) field_outputs.setdefault(field_number, []).extend(outputs) output_len += len(outputs) if output_len > 0: if ( config.preserve_field_order and field_order is not None and len(field_order) == output_len ): # check for old typedefs which had field_order as a tuple if isinstance(field_order[0], tuple): field_order = [x[0] for x in field_order] for field_number in field_order: try: output += field_outputs[field_number].pop(0) except (IndexError, KeyError): # If these don't match up despite us checking the overall # length, then we probably have something weird going on # with field naming. # This might mean ordering is off from the original, but # should break real protobuf messages logger.warning( "The field_order list does not match the fields from _encode_message_field" ) # If we're hitting a mismatch between the field order and # what data we have, then just bail. We can encode the rest # normally break # Group together elements in an array for values in field_outputs.values(): for value in values: output += value return output def _encode_message_field(config, typedef, path, field_id, value): # type: (Config, TypeDef, List[str], str | int, Any) -> Tuple[str, List[bytes]] if not isinstance(field_id, six.text_type): field_key = six.ensure_text(str(field_id)) # type: str else: field_key = field_id fielddef_results = typedef.lookup_fielddef(field_key) if fielddef_results is None: raise EncoderException( "Provided field name/number %s is not valid" % (field_key), path, ) field_number, fielddef = fielddef_results field_path = path[:] field_path.append(str(field_number)) field_type = fielddef.lookup_field_type(field_key, config, field_path) if field_type is None: raise EncoderException( "Provided field name/number %s / %s is not valid" % (field_key, field_number), field_path, ) field_encoder = None # type: Callable[[Any], bytes] | None if isinstance(field_type, TypeDef): field_typedef = field_type field_type = "message" field_encoder = lambda data: encode_lendelim_message( data, config, field_typedef, path=field_path, field_order=fielddef.field_order, ) else: if field_type not in blackboxprotobuf.lib.types.ENCODERS: raise TypedefException("Unknown type: %s" % field_type) field_encoder = blackboxprotobuf.lib.types.ENCODERS[field_type] if field_encoder is None: raise TypedefException( "Encoder not implemented for %s" % field_type, field_path ) # Encode the tag tag = encode_tag( int(field_number), blackboxprotobuf.lib.types.WIRETYPES[field_type] ) outputs = [] try: # Repeated values we'll encode each one separately and add them to the outputs list # Packed values take in a list, but encode them into a single length # delimited field, so we handle those as a non-repeated value if isinstance(value, list) and not field_type.startswith("packed_"): for repeated in value: outputs.append(tag + field_encoder(repeated)) else: outputs.append(tag + field_encoder(value)) except EncoderException as exc: exc.set_path(field_path) six.reraise(*sys.exc_info()) return field_number, outputs def decode_message(buf, config, typedef=None, pos=0, end=None, depth=0, path=None): # type: (bytes, Config, Optional[TypeDef], int, Optional[int], int, Optional[List[str]]) -> Tuple[Message, TypeDef, List[str], int] """Decode a protobuf message with no length prefix""" if end is None: end = len(buf) if typedef is None: typedef = TypeDef() if path is None: path = [] output = {} # type: Message seen_repeated = {} # type: Dict[str, bool] mut_typedef = typedef.make_mutable() grouped_fields, field_order, pos = _group_by_number(buf, pos, end, path) for field_number, (wire_type, buffers) in grouped_fields.items(): # wire_type should already be validated by _group_by_number field_path = path[:] + [field_number] fielddef_pair = mut_typedef.lookup_fielddef_number( field_number ) # type: Optional[Tuple[str, FieldDef]] if fielddef_pair is None: fielddef = FieldDef(field_number) else: fielddef = fielddef_pair[1] # Decode messages (which may have multiple typedefs) or unknown length delimited fields if wire_type == wiretypes.LENGTH_DELIMITED and not isinstance( fielddef.lookup_field_type_number("0", config, field_path), six.string_types ): output_map, new_fielddef = _try_decode_lendelim_fields( buffers, fielddef, config, field_path ) # Merge length delim field into the output map for field_key, field_outputs in output_map.items(): output.setdefault(field_key, []).extend(field_outputs) seen_repeated[fielddef.name] = new_fielddef.seen_repeated mut_typedef.set_fielddef(field_number, new_fielddef) else: field_outputs, new_fielddef, field_alt_type_id = _decode_standard_field( wire_type, buffers, fielddef, config, path ) field_key = new_fielddef.field_key(field_alt_type_id) output.setdefault(field_key, []).extend(field_outputs) seen_repeated[fielddef.name] = new_fielddef.seen_repeated # Save the field typedef/type back to the typedef mut_typedef.set_fielddef(field_number, new_fielddef) _simplify_output(output, seen_repeated) return output, mut_typedef, field_order, pos def _decode_standard_field(wire_type, buffers, fielddef, config, field_path): # type: (int, List[bytes], FieldDef, Config, List[str]) -> Tuple[List[Any], FieldDef, str] field_outputs = None field_alt_type_id = None for alt_type_id, field_type in fielddef.resolve_types(config, field_path).items(): if isinstance(field_type, TypeDef): # Skip message types continue if ( not isinstance(field_type, six.string_types) or blackboxprotobuf.lib.types.WIRETYPES[field_type] != wire_type ): raise DecoderException( "Type %s from typedef did not match wiretype %s" % (field_type, wire_type), path=field_path, ) if field_type not in blackboxprotobuf.lib.types.DECODERS: raise TypedefException( "Type %s does not have a decoder" % (field_type), path=field_path, ) decoder = blackboxprotobuf.lib.types.DECODERS[field_type] try: field_outputs = [decoder(buf, 0)[0] for buf in buffers] field_alt_type_id = alt_type_id except BlackboxProtobufException as exc: # Error decoding, try next one if we have one continue # Decoding worked break if field_outputs is None: field_type = config.get_default_type(wire_type) default_decoder = blackboxprotobuf.lib.types.DECODERS[field_type] field_outputs = [default_decoder(buf, 0)[0] for buf in buffers] mut_fielddef = fielddef.make_mutable() if field_alt_type_id is None: field_alt_type_id = mut_fielddef.next_alt_type_id() mut_fielddef.set_type(field_alt_type_id, field_type) if field_outputs is None: raise DecoderException( "Unable to decode wire_type %s" % (wire_type), path=field_path, ) if isinstance(field_type, six.string_types) and field_type.startswith("packed_"): # Packed decoding will return a list of lists field_outputs = [y for x in field_outputs for y in x] mut_fielddef.mark_repeated() # Mark repeated if we have have more than one # Don't need to worry if it's already repeated elif len(field_outputs) > 1: mut_fielddef.mark_repeated() return field_outputs, mut_fielddef, field_alt_type_id def _simplify_output(output, seen_repeated): # type: (Message, Dict[str, bool]) -> None # If any outputs only have one element, convert them from a list to solo # Mutates output for field_key, field_outputs in output.items(): if isinstance(field_outputs, list) and len(field_outputs) == 1: field_name = ( field_key.split(six.u("-"), 1)[0] if isinstance(field_key, six.string_types) else six.ensure_text(str(field_key)) ) if not seen_repeated[field_name]: output[field_key] = field_outputs[0] def _group_by_number(buf, pos, end, path): # type: (bytes, int, int, List[str]) -> Tuple[Dict[str, Tuple[int, List[bytes]]], List[str], int] # Parse through the whole message and split into buffers based on wire # type and organized by field number. This forces us to parse the whole # message at once, but I think we're doing that anyway. This catches size # errors early as well, which is usually the best indicator of if it's a # protobuf message or not. # Returns a dictionary like: # { # "2": (, []) # } output_map = {} # type: Dict[str, Tuple[int, List[bytes]]] field_order = [] while pos < end: # Read in a field field_number, wire_type, pos = decode_tag(buf, pos) # We want field numbers as strings everywhere field_id = six.ensure_text(str(field_number)) field_path = path[:] + [field_id] if field_id in output_map and output_map[field_id][0] != wire_type: # This should never happen raise DecoderException( "Field %s has mistmatched wiretypes. Previous: %s Now: %s" % (field_id, output_map[field_id][0], wire_type), path=field_path, ) length = None if wire_type == wiretypes.VARINT: # We actually have to read in the whole varint to figure out it's size _, new_pos = varint.decode_uvarint(buf, pos) length = new_pos - pos elif wire_type == wiretypes.FIXED32: length = 4 elif wire_type == wiretypes.FIXED64: length = 8 elif wire_type == wiretypes.LENGTH_DELIMITED: # Read the length from the start of the message # add on the length of the length tag as well bytes_length, new_pos = varint.decode_varint(buf, pos) length = bytes_length + (new_pos - pos) elif wire_type in [ wiretypes.START_GROUP, wiretypes.END_GROUP, ]: raise DecoderException("GROUP wire types not supported", path=field_path) else: raise DecoderException( "Got unknown wire type: %d" % wire_type, path=field_path ) if pos + length > end: raise DecoderException( "Decoded length for field %s goes over end: %d > %d" % (field_id, pos + length, end), path=field_path, ) field_buf = buf[pos : pos + length] if field_id in output_map: output_map[field_id][1].append(field_buf) else: output_map[field_id] = (wire_type, [field_buf]) field_order.append(field_id) pos += length return output_map, field_order, pos def _try_decode_lendelim_fields(buffers, fielddef, config, path): # type: (List[bytes], FieldDef, Config, List[str]) -> Tuple[Message, FieldDef] # Mutates message_output # This is where things get weird # To start, since we want to decode messages and not treat every # embedded message as bytes, we have to guess if it's a message or # not. # Unlike other types, we can't assume our message types are # consistent across the tree or even within the same message. # A field could be a bytes type that that decodes to multiple different # messages that don't have the same type definition. This is where # 'alt_typedefs' let us say that these are the different message types # we've seen for this one field. # In general, if something decodes as a message once, the rest should too # and we can enforce that across a single message, but not multiple # messages. # This is going to change the definition of "alt_typedefs" a bit from just # alternate message type definitions to also allowing downgrading to # 'bytes' or string with an 'alt_type' if it doesn't parse message_output = {} # type: Message # TODO potential performance improvement: Do a first pass with {} or just group by number and use the results to validate if it's even valid protobuf and quick match wire_types against typedefs try: outputs_map = {} # type: Dict[str, Any] field_order = [] # type: List[str] next_alt_type_id = int(fielddef.next_alt_type_id()) field_types = fielddef.resolve_types(config, path) # We don't want any mutable changes within this loop, we want # everything to rollback if it fails for buf in buffers: output = None output_typedef = None output_typedef_num = None new_field_order = [] # type: List[str] for alt_type_id, field_type in sorted( field_types.items(), key=lambda x: int(x[0]) ): # Skip non message types if not isinstance(field_type, TypeDef): continue try: ( output, output_typedef, new_field_order, _, ) = decode_lendelim_message(buf, config, field_type) except Exception as exc: # If we get an exception, then this isn't the right typedef, try the next continue output_typedef_num = alt_type_id # If we didn't get an exception, then we found the right type break # If we didn't find a type above, then try an anonymous type # If this fails, we fall back to string and bytes for all types if output is None: output, output_typedef, new_field_order, _ = decode_lendelim_message( buf, config, None ) output_typedef_num = six.ensure_text(str(next_alt_type_id)) next_alt_type_id += 1 if output_typedef is None or output_typedef_num is None: raise DecoderException( "Could not find an output_typedef or output_typedef_num. This should not happen under any circumstances." ) # save the output or typedef we found field_types[output_typedef_num] = output_typedef outputs_map.setdefault(output_typedef_num, []).append(output) # we should technically have a different field order for each instance of the data # but that would require a very messy JSON which we're trying to avoid if len(new_field_order) > len(field_order): field_order = new_field_order # was able to decode everything as a message mut_fielddef = fielddef.make_mutable() mut_fielddef.set_types(field_types) if config.preserve_field_order: mut_fielddef.set_field_order(field_order) # messages get set as "key-alt_number" for output_typedef_num, outputs in outputs_map.items(): output_field_key = mut_fielddef.field_key(output_typedef_num) message_output[output_field_key] = outputs if len(outputs) > 1: mut_fielddef.mark_repeated() # success, return return message_output, mut_fielddef except DecoderException as exc: # this should be pretty common, don't be noisy or throw an exception logger.debug( "Could not decode a buffer for field (%s) as a message: %s", path, exc, ) # Decoding as a message did not work, try strings and then the configured binary type # By default, default_binary_type will be redundant with bytes, but we want # to fall back on bytes if default_binary_type fails for any reason for target_type in ["string", config.default_binary_type, "bytes"]: try: outputs = [] decoder = blackboxprotobuf.lib.types.DECODERS[target_type] for buf in buffers: output, _ = decoder(buf, 0) outputs.append(output) field_alt_type_id = None # check if the type is already known field_types = fielddef.resolve_types(config, path) for alt_type_id, field_type in field_types.items(): if field_type == target_type: field_alt_type_id = alt_type_id break mut_fielddef = fielddef.make_mutable() if field_alt_type_id is None: field_alt_type_id = mut_fielddef.add_type(target_type) field_key = mut_fielddef.field_key(field_alt_type_id) # type: str message_output[field_key] = outputs return message_output, mut_fielddef except DecoderException: continue # This should never happen, we should always be able to use bytes raise DecoderException("Unable to decode field with typedef", path=path) def encode_lendelim_message(data, config, typedef, path=None, field_order=None): # type: (Message, Config, TypeDef, Optional[List[str]], Optional[List[str]]) -> bytes """Encode data as a length delimited protobuf message""" message_out = encode_message( data, config, typedef, path=path, field_order=field_order ) length = varint.encode_varint(len(message_out)) return length + message_out def decode_lendelim_message(buf, config, typedef=None, pos=0, depth=0, path=None): # type: (bytes, Config, Optional[TypeDef], int, int, Optional[List[str]]) -> Tuple[Message, TypeDef, List[str], int] """Deocde a length delimited protobuf message from buf""" length, pos = varint.decode_varint(buf, pos) ret = decode_message( buf, config, typedef, pos, pos + length, depth=depth, path=path ) return ret def generate_packed_encoder(wrapped_encoder): # type: (Callable[[Any], bytes]) -> Callable[[List[Any]], bytes] """Generate an encoder for a packed type based on a base type encoder""" def length_wrapper(values): # type: (List[Any]) -> bytes # Encode repeat values and prefix with the length output = bytearray() for value in values: output += wrapped_encoder(value) length = varint.encode_varint(len(output)) return length + output return length_wrapper def generate_packed_decoder(wrapped_decoder): # type: (Callable[[bytes, int], Tuple[Any, int]]) -> Callable[[bytes, int], Tuple[List[Any], int]] """Generate an decoder for a packed type based on a base type decoder""" def length_wrapper(buf, pos): # type: (bytes, int) -> Tuple[List[Any], int] # Decode repeat values prefixed with the length length, pos = varint.decode_varint(buf, pos) end = pos + length output = [] while pos < end: value, pos = wrapped_decoder(buf, pos) output.append(value) if pos > end: raise DecoderException( ( "Error decoding packed field. Packed length larger than" " buffer: decoded = %d, left = %d" ) % (length, len(buf) - pos) ) return output, pos return length_wrapper blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/types/type_maps.py000066400000000000000000000123141476504373500255040ustar00rootroot00000000000000"""Contains various maps for protobuf types, including encoding/decoding functions, wiretypes and default types """ # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from blackboxprotobuf.lib.types import varint, fixed, length_delim, wiretypes import six if six.PY3: from typing import Any, Callable, Dict, Tuple # Map a blackboxprotobuf type to specific encoder ENCODERS = { "uint": varint.encode_uvarint, "int": varint.encode_varint, "sint": varint.encode_svarint, "fixed32": fixed.encode_fixed32, "sfixed32": fixed.encode_sfixed32, "float": fixed.encode_float, "fixed64": fixed.encode_fixed64, "sfixed64": fixed.encode_sfixed64, "double": fixed.encode_double, "bytes": length_delim.encode_bytes, "bytes_hex": length_delim.encode_bytes_hex, "string": length_delim.encode_string, "packed_uint": length_delim.generate_packed_encoder(varint.encode_uvarint), "packed_int": length_delim.generate_packed_encoder(varint.encode_varint), "packed_sint": length_delim.generate_packed_encoder(varint.encode_svarint), "packed_fixed32": length_delim.generate_packed_encoder(fixed.encode_fixed32), "packed_sfixed32": length_delim.generate_packed_encoder(fixed.encode_sfixed32), "packed_float": length_delim.generate_packed_encoder(fixed.encode_float), "packed_fixed64": length_delim.generate_packed_encoder(fixed.encode_fixed64), "packed_sfixed64": length_delim.generate_packed_encoder(fixed.encode_sfixed64), "packed_double": length_delim.generate_packed_encoder(fixed.encode_double), } # type: Dict[str, Callable[[Any], bytes]] # Map a blackboxprotobuf type to specific decoder DECODERS = { "uint": varint.decode_uvarint, "int": varint.decode_varint, "sint": varint.decode_svarint, "fixed32": fixed.decode_fixed32, "sfixed32": fixed.decode_sfixed32, "float": fixed.decode_float, "fixed64": fixed.decode_fixed64, "sfixed64": fixed.decode_sfixed64, "double": fixed.decode_double, "bytes": length_delim.decode_bytes, "bytes_hex": length_delim.decode_bytes_hex, "string": length_delim.decode_string, "packed_uint": length_delim.generate_packed_decoder(varint.decode_uvarint), "packed_int": length_delim.generate_packed_decoder(varint.decode_varint), "packed_sint": length_delim.generate_packed_decoder(varint.decode_svarint), "packed_fixed32": length_delim.generate_packed_decoder(fixed.decode_fixed32), "packed_sfixed32": length_delim.generate_packed_decoder(fixed.decode_sfixed32), "packed_float": length_delim.generate_packed_decoder(fixed.decode_float), "packed_fixed64": length_delim.generate_packed_decoder(fixed.decode_fixed64), "packed_sfixed64": length_delim.generate_packed_decoder(fixed.decode_sfixed64), "packed_double": length_delim.generate_packed_decoder(fixed.decode_double), } # type: Dict[str, Callable[[bytes, int], Tuple[Any, int] ]] WIRETYPES = { "uint": wiretypes.VARINT, "int": wiretypes.VARINT, "sint": wiretypes.VARINT, "fixed32": wiretypes.FIXED32, "sfixed32": wiretypes.FIXED32, "float": wiretypes.FIXED32, "fixed64": wiretypes.FIXED64, "sfixed64": wiretypes.FIXED64, "double": wiretypes.FIXED64, "bytes": wiretypes.LENGTH_DELIMITED, "bytes_hex": wiretypes.LENGTH_DELIMITED, "string": wiretypes.LENGTH_DELIMITED, "message": wiretypes.LENGTH_DELIMITED, "group": wiretypes.START_GROUP, "packed_uint": wiretypes.LENGTH_DELIMITED, "packed_int": wiretypes.LENGTH_DELIMITED, "packed_sint": wiretypes.LENGTH_DELIMITED, "packed_fixed32": wiretypes.LENGTH_DELIMITED, "packed_sfixed32": wiretypes.LENGTH_DELIMITED, "packed_float": wiretypes.LENGTH_DELIMITED, "packed_fixed64": wiretypes.LENGTH_DELIMITED, "packed_sfixed64": wiretypes.LENGTH_DELIMITED, "packed_double": wiretypes.LENGTH_DELIMITED, } # type: Dict[str, int] # Default values to use when decoding each wire type # length delimited is special and handled in the length_delim module WIRE_TYPE_DEFAULTS = { wiretypes.VARINT: "int", wiretypes.FIXED32: "fixed32", wiretypes.FIXED64: "fixed64", wiretypes.LENGTH_DELIMITED: None, wiretypes.START_GROUP: None, wiretypes.END_GROUP: None, } # type: Dict[int, str | None] blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/types/varint.py000066400000000000000000000147051476504373500250140ustar00rootroot00000000000000"""Classes for encoding and decoding varint types""" # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import binascii import struct import six from blackboxprotobuf.lib.exceptions import EncoderException, DecoderException if six.PY3: from typing import Any, Tuple # These are set in decoder.py # In theory, uvarints and zigzag varints shouldn't have a max # But this is enforced by protobuf MAX_UVARINT = (1 << 64) - 1 MIN_UVARINT = 0 MAX_SVARINT = (1 << 63) - 1 MIN_SVARINT = -(1 << 63) def encode_uvarint(value): # type: (Any) -> bytes """Encode a long or int into a bytearray.""" if not isinstance(value, six.integer_types): raise EncoderException("Got non-int type for uvarint encoding: %s" % value) output = bytearray() if value < MIN_UVARINT: raise EncoderException( "Error encoding %d as uvarint. Value must be positive" % value ) if value > MAX_UVARINT: raise EncoderException( "Error encoding %d as uvarint. Value must be %s or less" % (value, MAX_UVARINT) ) if not value: output.append(value & 0x7F) else: while value: next_byte = value & 0x7F value >>= 7 if value: next_byte |= 0x80 output.append(next_byte) return output def decode_uvarint(buf, pos): # type: (bytes, int) -> Tuple[int, int] """Decode bytearray into a long.""" pos_start = pos # Convert buffer to string if six.PY2: buf = bytes(buf) try: value = 0 shift = 0 while six.indexbytes(buf, pos) & 0x80: value += (six.indexbytes(buf, pos) & 0x7F) << (shift * 7) pos += 1 shift += 1 value += (six.indexbytes(buf, pos) & 0x7F) << (shift * 7) pos += 1 except IndexError: raise DecoderException( "Error decoding uvarint: read past the end of the buffer" ) # Validate that this is a cononical encoding by re-encoding the value try: test_encode = encode_uvarint(value) except EncoderException as ex: raise DecoderException( "Error decoding uvarint: value (%s) was not able to be re-encoded: %s" % (value, ex) ) if buf[pos_start:pos] != test_encode: raise DecoderException( "Error decoding uvarint: Encoding is not standard:\noriginal: %r\nstandard: %r" % (buf[pos_start:pos], test_encode) ) return (value, pos) def encode_varint(value): # type: (Any) -> bytes """Encode a long or int into a bytearray.""" if not isinstance(value, six.integer_types): raise EncoderException("Got non-int type for varint encoding: %s" % value) if value > MAX_SVARINT: raise EncoderException( "Error encoding %d as varint. Value must be <= %s" % (value, MAX_SVARINT) ) if value < MIN_SVARINT: raise EncoderException( "Error encoding %d as varint. Value must be >= %s" % (value, MIN_SVARINT) ) if value < 0: value += 1 << 64 output = encode_uvarint(value) return output def decode_varint(buf, pos): # type: (bytes, int) -> Tuple[int, int] """Decode bytearray into a long.""" # Convert buffer to string pos_start = pos if six.PY2: buf = bytes(buf) value, pos = decode_uvarint(buf, pos) if value & (1 << 63): value -= 1 << 64 # Validate that this is a cononical encoding by re-encoding the value try: test_encode = encode_varint(value) except EncoderException as ex: raise DecoderException( "Error decoding varint: value (%s) was not able to be re-encoded: %s" % (value, ex) ) if buf[pos_start:pos] != test_encode: raise DecoderException( "Error decoding varint: Encoding is not standard:\noriginal: %r\nstandard: %r" % (buf[pos_start:pos], test_encode) ) return (value, pos) def encode_zig_zag(value): # type: (int) -> int if value < 0: return (abs(value) << 1) - 1 return value << 1 def decode_zig_zag(value): # type: (int) -> int if value & 0x1: # negative return -((value + 1) >> 1) return value >> 1 def encode_svarint(value): # type: (Any) -> bytes """Zigzag encode the potentially signed value prior to encoding""" if not isinstance(value, six.integer_types): raise EncoderException("Got non-int type for svarint encoding: %s" % value) # zigzag encode value if value > MAX_SVARINT: raise EncoderException( "Error encoding %d as svarint. Value must be <= %s" % (value, MAX_SVARINT) ) if value < MIN_SVARINT: raise EncoderException( "Error encoding %d as svarint. Value must be >= %s" % (value, MIN_SVARINT) ) return encode_uvarint(encode_zig_zag(value)) def decode_svarint(buf, pos): # type: (bytes, int) -> Tuple[int, int] """Decode bytearray into a long.""" pos_start = pos output, pos = decode_uvarint(buf, pos) value = decode_zig_zag(output) # Validate that this is a cononical encoding by re-encoding the value test_encode = encode_svarint(value) if buf[pos_start:pos] != test_encode: raise DecoderException( "Error decoding svarint: Encoding is not standard:\noriginal: %r\nstandard: %r" % (buf[pos_start:pos], test_encode) ) return value, pos blackboxprotobuf-1.4.2/lib/blackboxprotobuf/lib/types/wiretypes.py000066400000000000000000000023011476504373500255310ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # Define the wiretype int values VARINT = 0 FIXED64 = 1 LENGTH_DELIMITED = 2 START_GROUP = 3 END_GROUP = 4 FIXED32 = 5 blackboxprotobuf-1.4.2/lib/blackboxprotobuf/py.typed000066400000000000000000000000001476504373500227030ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/poetry.lock000066400000000000000000000476001476504373500200530ustar00rootroot00000000000000# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] dev = ["attrs[docs,tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] [[package]] name = "black" version = "23.11.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ {file = "black-23.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dbea0bb8575c6b6303cc65017b46351dc5953eea5c0a59d7b7e3a2d2f433a911"}, {file = "black-23.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:412f56bab20ac85927f3a959230331de5614aecda1ede14b373083f62ec24e6f"}, {file = "black-23.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d136ef5b418c81660ad847efe0e55c58c8208b77a57a28a503a5f345ccf01394"}, {file = "black-23.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:6c1cac07e64433f646a9a838cdc00c9768b3c362805afc3fce341af0e6a9ae9f"}, {file = "black-23.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cf57719e581cfd48c4efe28543fea3d139c6b6f1238b3f0102a9c73992cbb479"}, {file = "black-23.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:698c1e0d5c43354ec5d6f4d914d0d553a9ada56c85415700b81dc90125aac244"}, {file = "black-23.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:760415ccc20f9e8747084169110ef75d545f3b0932ee21368f63ac0fee86b221"}, {file = "black-23.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:58e5f4d08a205b11800332920e285bd25e1a75c54953e05502052738fe16b3b5"}, {file = "black-23.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:45aa1d4675964946e53ab81aeec7a37613c1cb71647b5394779e6efb79d6d187"}, {file = "black-23.11.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c44b7211a3a0570cc097e81135faa5f261264f4dfaa22bd5ee2875a4e773bd6"}, {file = "black-23.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a9acad1451632021ee0d146c8765782a0c3846e0e0ea46659d7c4f89d9b212b"}, {file = "black-23.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc7f6a44d52747e65a02558e1d807c82df1d66ffa80a601862040a43ec2e3142"}, {file = "black-23.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7f622b6822f02bfaf2a5cd31fdb7cd86fcf33dab6ced5185c35f5db98260b055"}, {file = "black-23.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:250d7e60f323fcfc8ea6c800d5eba12f7967400eb6c2d21ae85ad31c204fb1f4"}, {file = "black-23.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5133f5507007ba08d8b7b263c7aa0f931af5ba88a29beacc4b2dc23fcefe9c06"}, {file = "black-23.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:421f3e44aa67138ab1b9bfbc22ee3780b22fa5b291e4db8ab7eee95200726b07"}, {file = "black-23.11.0-py3-none-any.whl", hash = "sha256:54caaa703227c6e0c87b76326d0862184729a69b73d3b7305b6288e1d830067e"}, {file = "black-23.11.0.tar.gz", hash = "sha256:4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05"}, ] [package.dependencies] click = ">=8.0.0" mypy-extensions = ">=0.4.3" packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "click" version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] [[package]] name = "exceptiongroup" version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] test = ["pytest (>=6)"] [[package]] name = "hypothesis" version = "6.91.0" description = "A library for property-based testing" optional = false python-versions = ">=3.8" files = [ {file = "hypothesis-6.91.0-py3-none-any.whl", hash = "sha256:316e06d6f7d5f8ab87bcc7417fca750a2b082ed3ce902b979816b413276680b3"}, {file = "hypothesis-6.91.0.tar.gz", hash = "sha256:a9f61a2bcfc342febcc1d04b80a99e789c57b700f91cbd43bbdb5d651af385cd"}, ] [package.dependencies] attrs = ">=19.2.0" exceptiongroup = {version = ">=1.0.0", markers = "python_version < \"3.11\""} sortedcontainers = ">=2.1.0,<3.0.0" [package.extras] all = ["backports.zoneinfo (>=0.2.1)", "black (>=19.10b0)", "click (>=7.0)", "django (>=3.2)", "dpcontracts (>=0.4)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.17.3)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2023.3)"] cli = ["black (>=19.10b0)", "click (>=7.0)", "rich (>=9.0.0)"] codemods = ["libcst (>=0.3.16)"] dateutil = ["python-dateutil (>=1.4)"] django = ["django (>=3.2)"] dpcontracts = ["dpcontracts (>=0.4)"] ghostwriter = ["black (>=19.10b0)"] lark = ["lark (>=0.10.1)"] numpy = ["numpy (>=1.17.3)"] pandas = ["pandas (>=1.1)"] pytest = ["pytest (>=4.6)"] pytz = ["pytz (>=2014.1)"] redis = ["redis (>=3.0.0)"] zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2023.3)"] [[package]] name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] [[package]] name = "mypy" version = "1.10.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ {file = "mypy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2"}, {file = "mypy-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99"}, {file = "mypy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2"}, {file = "mypy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9"}, {file = "mypy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051"}, {file = "mypy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1"}, {file = "mypy-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee"}, {file = "mypy-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de"}, {file = "mypy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7"}, {file = "mypy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53"}, {file = "mypy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b"}, {file = "mypy-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30"}, {file = "mypy-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e"}, {file = "mypy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5"}, {file = "mypy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda"}, {file = "mypy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0"}, {file = "mypy-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727"}, {file = "mypy-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4"}, {file = "mypy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061"}, {file = "mypy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f"}, {file = "mypy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976"}, {file = "mypy-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec"}, {file = "mypy-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821"}, {file = "mypy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746"}, {file = "mypy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a"}, {file = "mypy-1.10.0-py3-none-any.whl", hash = "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee"}, {file = "mypy-1.10.0.tar.gz", hash = "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131"}, ] [package.dependencies] mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] install-types = ["pip"] mypyc = ["setuptools (>=50)"] reports = ["lxml"] [[package]] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.5" files = [ {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] [[package]] name = "packaging" version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.7" files = [ {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] [[package]] name = "platformdirs" version = "4.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.7" files = [ {file = "platformdirs-4.0.0-py3-none-any.whl", hash = "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b"}, {file = "platformdirs-4.0.0.tar.gz", hash = "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731"}, ] [package.extras] docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] [[package]] name = "pluggy" version = "1.3.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] [package.extras] dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" version = "3.20.3" description = "Protocol Buffers" optional = false python-versions = ">=3.7" files = [ {file = "protobuf-3.20.3-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99"}, {file = "protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e"}, {file = "protobuf-3.20.3-cp310-cp310-win32.whl", hash = "sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c"}, {file = "protobuf-3.20.3-cp310-cp310-win_amd64.whl", hash = "sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7"}, {file = "protobuf-3.20.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469"}, {file = "protobuf-3.20.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4"}, {file = "protobuf-3.20.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4"}, {file = "protobuf-3.20.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454"}, {file = "protobuf-3.20.3-cp37-cp37m-win32.whl", hash = "sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905"}, {file = "protobuf-3.20.3-cp37-cp37m-win_amd64.whl", hash = "sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c"}, {file = "protobuf-3.20.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7"}, {file = "protobuf-3.20.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee"}, {file = "protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050"}, {file = "protobuf-3.20.3-cp38-cp38-win32.whl", hash = "sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86"}, {file = "protobuf-3.20.3-cp38-cp38-win_amd64.whl", hash = "sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9"}, {file = "protobuf-3.20.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b"}, {file = "protobuf-3.20.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b"}, {file = "protobuf-3.20.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402"}, {file = "protobuf-3.20.3-cp39-cp39-win32.whl", hash = "sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480"}, {file = "protobuf-3.20.3-cp39-cp39-win_amd64.whl", hash = "sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7"}, {file = "protobuf-3.20.3-py2.py3-none-any.whl", hash = "sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db"}, {file = "protobuf-3.20.3.tar.gz", hash = "sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2"}, ] [[package]] name = "pytest" version = "7.4.3" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, ] [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] [[package]] name = "sortedcontainers" version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" optional = false python-versions = "*" files = [ {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] [[package]] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.7" files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] [[package]] name = "typing-extensions" version = "4.8.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, ] [metadata] lock-version = "2.0" python-versions = "^3.8" content-hash = "9a2bd7bf0ff93ad3ad7a15b059cb197dc6a991c55dc870395c50bfff64e1cd01" blackboxprotobuf-1.4.2/lib/pyproject.toml000066400000000000000000000014121476504373500205620ustar00rootroot00000000000000[tool.poetry] name = "bbpb" version = "1.4.2" description = "Library for working with protobuf messages without a protobuf type definition." authors = ["Ryan Winkelmaier "] license = "MIT" repository = "https://github.com/nccgroup/blackboxprotobuf" readme = "README.md" keywords = ["protobuf"] exclude = ["./tests"] packages = [ { include = "blackboxprotobuf" } ] [tool.poetry.dependencies] python = "^3.8" six = "^1.16" [tool.poetry.dev-dependencies] pytest = "^7.4.2" hypothesis = "^6.31.6" black = "^23.9.1" protobuf = "^3.20" [tool.poetry.group.dev.dependencies] mypy = "^1.10.0" [tool.poetry.scripts] bbpb = "blackboxprotobuf.__main__:main" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" blackboxprotobuf-1.4.2/lib/tests/000077500000000000000000000000001476504373500170125ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/tests/deps/000077500000000000000000000000001476504373500177455ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/tests/deps/protobuf/000077500000000000000000000000001476504373500216055ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/tests/generate_payload.sh000066400000000000000000000001301476504373500226430ustar00rootroot00000000000000#!/bin/bash protoc --encode=TestMessage payloads/Test.proto < payloads/test_message.in blackboxprotobuf-1.4.2/lib/tests/payloads/000077500000000000000000000000001476504373500206265ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/tests/payloads/Test.proto000066400000000000000000000013031476504373500226270ustar00rootroot00000000000000syntax = "proto3"; // Set up a protobuf message that contains every possible type message TestMessage { double testDouble = 1; float testFloat = 2; int32 testInt32 = 4; int64 testInt64 = 8; uint32 testUInt32 = 16; uint64 testUInt64 = 32; sint32 testSInt32 = 64; sint64 testSInt64 = 128; fixed32 testFixed32 = 256; fixed64 testFixed64 = 512; sfixed32 testSFixed32 = 1024; sfixed64 testSFixed64 = 2048; bool testBool = 4096; string testString = 8192; bytes testBytes = 16384; EmbeddedMessage testEmbed = 32768; repeated int32 testRepeatedInt32 = 65536; } message EmbeddedMessage { double embedDouble = 3; string embedString = 2; } blackboxprotobuf-1.4.2/lib/tests/payloads/test_message.in000066400000000000000000000005461476504373500236460ustar00rootroot00000000000000testDouble: 4.55 testFloat: 3.14 testInt32: 5 testInt64: 2 testUInt32: 9 testUInt64: 1024 testSInt32: -2 testSInt64: -1 testFixed32: 20 testFixed64: 10 testSFixed32: -20 testSFixed64: -34 testBool: 1 testString: "Test\n String" testBytes: "bytes\0aaa" testEmbed: { embedDouble: 2.1 embedString: "Test1234" } testRepeatedInt32: 2 testRepeatedInt32: 5 blackboxprotobuf-1.4.2/lib/tests/payloads/test_message.out000066400000000000000000000001761476504373500240460ustar00rootroot00000000000000 333333@ÃõH@ @€ €€€€… …@ìÿÿÿ€Þÿÿÿÿÿÿÿ€€‚€ Test String‚€ bytesaaa‚€Test1234ÍÌÌÌÌÌ@‚€ blackboxprotobuf-1.4.2/lib/tests/proxy_tests/000077500000000000000000000000001476504373500214155ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/tests/proxy_tests/Makefile000066400000000000000000000002651476504373500230600ustar00rootroot00000000000000protoc: python3 -m grpc_tools.protoc -I./ --python_out=./ --grpc_python_out=./ Test.proto certs: openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes blackboxprotobuf-1.4.2/lib/tests/proxy_tests/Test.proto000066400000000000000000000014221476504373500234200ustar00rootroot00000000000000syntax = "proto3"; // Set up a protobuf message that contains every possible type message TestMessage { double testDouble = 1; float testFloat = 2; int32 testInt32 = 4; int64 testInt64 = 8; uint32 testUInt32 = 16; uint64 testUInt64 = 32; sint32 testSInt32 = 64; sint64 testSInt64 = 128; fixed32 testFixed32 = 256; fixed64 testFixed64 = 512; sfixed32 testSFixed32 = 1024; sfixed64 testSFixed64 = 2048; bool testBool = 4096; string testString = 8192; bytes testBytes = 16384; EmbeddedMessage testEmbed = 32768; repeated int32 testRepeatedInt32 = 65536; } message EmbeddedMessage { double embedDouble = 3; string embedString = 2; } service TestService { rpc TestRPC(TestMessage) returns (TestMessage) {} } blackboxprotobuf-1.4.2/lib/tests/proxy_tests/grpc_client.py000066400000000000000000000031411476504373500242570ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import grpc import Test_pb2 import Test_pb2_grpc grpc_options = [ ("grpc.enable_http_proxy", True), ("grpc.http_proxy", "http://127.0.0.1:8080"), ] with open("proxy_cert.pem", "rb") as f: ssl_cert = f.read() credentials = grpc.ssl_channel_credentials(ssl_cert) with grpc.secure_channel( "127.0.0.1:8000", credentials, options=grpc_options ) as channel: stub = Test_pb2_grpc.TestServiceStub(channel) response = stub.TestRPC(Test_pb2.TestMessage(testString="test123")) print("Got response: %s" % response) blackboxprotobuf-1.4.2/lib/tests/proxy_tests/grpc_server.py000066400000000000000000000034321476504373500243120ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import grpc from concurrent import futures import Test_pb2_grpc class TestService(Test_pb2_grpc.TestService): def TestRPC(self, msg, ctx): print("Got RPC message: %s" % msg) return msg def serve(): with open("key.pem", "rb") as f: ssl_key = f.read() with open("cert.pem", "rb") as f: ssl_cert = f.read() credentials = grpc.ssl_server_credentials([(ssl_key, ssl_cert)]) server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) Test_pb2_grpc.add_TestServiceServicer_to_server(TestService(), server) server.add_secure_port("127.0.0.1:8000", credentials) server.start() server.wait_for_termination() if __name__ == "__main__": serve() blackboxprotobuf-1.4.2/lib/tests/proxy_tests/http_client.py000066400000000000000000000051511476504373500243060ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import requests import zlib import Test_pb2 import struct for payload_type in ["none", "gzip", "grpc"]: message = Test_pb2.TestMessage(testString="test123").SerializeToString() print(f"Sending payload encoded with {payload_type}") if payload_type == "gzip": message = zlib.compress(message, level=9, wbits=31) elif payload_type == "grpc": # Fake grpc wrapper length = len(message) old_message = message message = bytearray() message.append(0x00) message.extend(struct.pack(">I", length)) message.extend(old_message) response = requests.post( "http://localhost:8000", data=message, headers={ "content-type": "application/protobuf", "payload_encoding": payload_type, }, proxies={"http": "http://localhost:8080"}, ) print(f"Got response: {response.status_code} {response.text}") response_content = response.content if payload_type == "gzip": response_content = zlib.decompress(response_content, wbits=31) elif payload_type == "grpc": old_response_content = response_content compression_byte = response_content[0] assert compression_byte == 0 length = struct.unpack_from(">I", response_content[1:])[0] response_content = old_response_content[5:] assert length == len(response_content) response_message = Test_pb2.TestMessage() response_message.ParseFromString(response_content) print(f"Got response message: {response_message}") blackboxprotobuf-1.4.2/lib/tests/proxy_tests/http_server.py000066400000000000000000000051631476504373500243410ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import Test_pb2 from http.server import BaseHTTPRequestHandler, HTTPServer import zlib import struct class TestHandler(BaseHTTPRequestHandler): def do_POST(self): payload_type = self.headers.get("payload_encoding", "none") print(f"Got connection with payload encoding: {payload_type}") data = self.rfile.read1() if payload_type == "gzip": data = zlib.decompress(data, wbits=31) elif payload_type == "grpc": old_data = data compression_byte = data[0] assert compression_byte == 0 length = struct.unpack_from(">I", data[1:])[0] data = old_data[5:] assert length == len(data) print("Got data: %s" % data) message = Test_pb2.TestMessage() message.ParseFromString(data) print("Got message: %s" % data) output = message.SerializeToString() if payload_type == "gzip": output = zlib.compress(output, level=9, wbits=31) elif payload_type == "grpc": # Fake grpc wrapper length = len(output) old_output = output output = bytearray() output.append(0x00) output.extend(struct.pack(">I", length)) output.extend(old_output) self.send_response(200) self.send_header("content-type", "application/protobuf") self.send_header("content-length", len(output)) self.end_headers() self.wfile.write(output) server = HTTPServer(("", 8000), TestHandler) server.serve_forever() blackboxprotobuf-1.4.2/lib/tests/proxy_tests/requirements.txt000066400000000000000000000000651476504373500247020ustar00rootroot00000000000000grpcio protobuf requests websockets websocket-client blackboxprotobuf-1.4.2/lib/tests/proxy_tests/websocket_client.py000066400000000000000000000047721476504373500253250ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import zlib import Test_pb2 import struct import websocket payload_type = "grpc" message = Test_pb2.TestMessage(testString="test123").SerializeToString() if payload_type == "gzip": message = zlib.compress(message, level=9, wbits=31) elif payload_type == "grpc": # Fake grpc wrapper length = len(message) old_message = message message = bytearray() message.append(0x00) message.extend(struct.pack(">I", length)) message.extend(old_message) ws = websocket.WebSocket() ws.connect( "ws://localhost:8000", proxy_type="http", http_proxy_host="localhost", http_proxy_port="8080", http_no_proxy=["test"], ) print("Connected") ws.send(bytes(message), websocket.ABNF.OPCODE_BINARY) print("Sent message") response_content = ws.recv() if len(response_content) == 0: print("Got an Empty response") else: if payload_type == "gzip": response_content = zlib.decompress(response_content, wbits=31) elif payload_type == "grpc": old_response_content = response_content compression_byte = response_content[0] assert compression_byte == 0 length = struct.unpack_from(">I", response_content[1:])[0] response_content = old_response_content[5:] assert length == len(response_content) response_message = Test_pb2.TestMessage() response_message.ParseFromString(response_content) print(f"Got response message: {response_message}") ws.close() blackboxprotobuf-1.4.2/lib/tests/proxy_tests/websocket_server.py000066400000000000000000000047611476504373500253530ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import Test_pb2 import zlib import struct import asyncio from websockets.server import serve payload_type = "grpc" async def handle_messages(websocket): async for message in websocket: print(f"Got message: {type(message)} {message}") data = message if payload_type == "gzip": data = zlib.decompress(data, wbits=31) elif payload_type == "grpc": old_data = data compression_byte = data[0] assert compression_byte == 0 length = struct.unpack_from(">I", data[1:])[0] data = old_data[5:] assert length == len(data) print("Got data: %s" % data) message = Test_pb2.TestMessage() message.ParseFromString(data) print("Got message: %s" % data) message.testString += "_server" output = message.SerializeToString() if payload_type == "gzip": output = zlib.compress(output, level=9, wbits=31) elif payload_type == "grpc": # Fake grpc wrapper length = len(output) old_output = output output = bytearray() output.append(0x00) output.extend(struct.pack(">I", length)) output.extend(old_output) await websocket.send(output) async def main(): async with serve(handle_messages, "localhost", 8000): await asyncio.Future() # run forever asyncio.run(main()) blackboxprotobuf-1.4.2/lib/tests/py_test/000077500000000000000000000000001476504373500205015ustar00rootroot00000000000000blackboxprotobuf-1.4.2/lib/tests/py_test/strategies.py000066400000000000000000000155411476504373500232330ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import six import binascii import hypothesis.strategies as st import blackboxprotobuf from blackboxprotobuf.lib.types import varint from blackboxprotobuf.lib.types import type_maps from hypothesis import settings from hypothesis import database from datetime import timedelta settings.register_profile( "quick", max_examples=100, database=database.ExampleDatabase(".hypothesis-db"), print_blob=True, ) settings.register_profile( "extended", max_examples=1000, database=database.ExampleDatabase(".hypothesis-db"), print_blob=True, ) settings.register_profile( "superextended", max_examples=10000, database=database.ExampleDatabase(".hypothesis-db"), print_blob=True, ) settings.load_profile("quick") @st.composite def message_typedef_gen(draw, max_depth=3, anon=False, types=None, named_fields=True): output = {} field_numbers = draw( st.lists(st.integers(min_value=1, max_value=2000).map(str), min_size=1) ) # pre-generate names so we can be sure they're unique field_names = draw( st.lists( st.from_regex(blackboxprotobuf.NAME_REGEX), min_size=len(field_numbers), max_size=len(field_numbers), unique_by=lambda x: x.lower(), ) ) if types is None: message_types = [ field_type for field_type in type_maps.WIRETYPES.keys() if field_type in input_map and input_map[field_type] is not None ] else: message_types = types for field_number, field_name in zip(field_numbers, field_names): field_number = six.ensure_text(str(field_number)) if max_depth == 0 and "message" in message_types: message_types.remove("message") field_type = draw(st.sampled_from(message_types)) output[field_number] = {} output[field_number]["type"] = field_type if field_type.startswith("packed"): output[field_number]["seen_repeated"] = True elif anon and field_type in [ "message", "string", "bytes", ]: # repeated fields can't be tested easily with anonymous types due to the type guessing output[field_number]["seen_repeated"] = False else: output[field_number]["seen_repeated"] = draw(st.booleans()) if field_type == "message": output[field_number]["message_typedef"] = draw( message_typedef_gen( max_depth=max_depth - 1, anon=anon, types=types, named_fields=named_fields, ) ) # decide whether to give it a name if named_fields and not anon and draw(st.booleans()): output[field_number]["name"] = six.ensure_text(field_name) return output @st.composite def gen_message_data(draw, type_def): output = {} for number, field in type_def.items(): if "name" in field and field["name"] != "": field_label = field["name"] else: field_label = six.ensure_text(str(number)) field_type = field["type"] if field_type == "message": strat = gen_message_data(field["message_typedef"]) else: strat = input_map[field["type"]] if field.get("seen_repeated", False) and not field_type.startswith("packed"): output[field_label] = draw(st.lists(strat, min_size=2)) else: output[field_label] = draw(strat) return output @st.composite # if anon is True, typedef will only have "default" types that it will decode # to without a typedef def gen_message(draw, anon=False, named_fields=True): allowed_types = None if anon: allowed_types = list( filter(lambda x: x is not None, type_maps.WIRE_TYPE_DEFAULTS.values()) ) # add length delim wiretypes allowed_types += ["message", "string", "bytes"] type_def = draw( message_typedef_gen(anon=anon, types=allowed_types, named_fields=named_fields) ) message = draw(gen_message_data(type_def)) return type_def, message # Map types to generators input_map = { "fixed32": st.integers(min_value=0, max_value=(2**32) - 1), "sfixed32": st.integers(min_value=-(2**31), max_value=(2**31) - 1), "fixed64": st.integers(min_value=0, max_value=(2**64) - 1), "sfixed64": st.integers(min_value=-(2**63), max_value=(2**63) - 1), "float": st.floats(width=32, allow_nan=False), "double": st.floats(width=64, allow_nan=False), "uint": st.integers(min_value=varint.MIN_UVARINT, max_value=varint.MAX_UVARINT), "int": st.integers(min_value=varint.MIN_SVARINT, max_value=varint.MAX_SVARINT), "sint": st.integers(min_value=varint.MIN_SVARINT, max_value=varint.MAX_SVARINT), "bytes": st.binary(), "string": st.text(), #'bytes_hex': st.binary().map(binascii.hexlify), "message": gen_message(), } input_map.update( { "packed_uint": st.lists(input_map["uint"], min_size=1), "packed_int": st.lists(input_map["int"], min_size=1), "packed_sint": st.lists(input_map["sint"], min_size=1), "packed_fixed32": st.lists(input_map["fixed32"], min_size=1), "packed_sfixed32": st.lists(input_map["sfixed32"], min_size=1), "packed_float": st.lists(input_map["float"], min_size=1), "packed_fixed64": st.lists(input_map["fixed64"], min_size=1), "packed_sfixed64": st.lists(input_map["sfixed64"], min_size=1), "packed_double": st.lists(input_map["double"], min_size=1), "packed_bytes": st.lists(input_map["bytes"], min_size=1), "packed_string": st.lists(input_map["string"], min_size=1), "packed_bytes": st.lists(input_map["bytes"], min_size=1), #'packed_bytes_hex': st.lists(input_map['bytes_hex'], min_size=1), } ) blackboxprotobuf-1.4.2/lib/tests/py_test/test_exceptions.py000066400000000000000000000167711476504373500243070ustar00rootroot00000000000000"""Try to test the exception generation by the library. Everything should throw some form of BlackboxProtobufException.""" # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from hypothesis import given import hypothesis.strategies as st from blackboxprotobuf.lib import config from blackboxprotobuf.lib.types import fixed, varint, length_delim from blackboxprotobuf.lib.exceptions import ( BlackboxProtobufException, DecoderException, EncoderException, ) # Fixed exception tests ## Encoding @given(value=st.integers()) def test_encode_fixed32(value): try: fixed.encode_fixed32(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(value=st.integers()) def test_encode_sfixed32(value): try: fixed.encode_sfixed32(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(value=st.decimals()) def test_encode_float(value): try: fixed.encode_float(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(value=st.integers()) def test_encode_fixed64(value): try: fixed.encode_fixed64(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(value=st.integers()) def test_encode_sfixed64(value): try: fixed.encode_sfixed64(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(value=st.decimals()) def test_encode_double(value): try: fixed.encode_double(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass ## Decoding @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) def test_decode_fixed32(buf, pos): try: fixed.decode_fixed32(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) def test_decode_sfixed32(buf, pos): try: fixed.decode_sfixed32(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) def test_decode_float(buf, pos): try: fixed.decode_float(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) def test_decode_fixed64(buf, pos): try: fixed.decode_fixed64(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) def test_decode_sfixed64(buf, pos): try: fixed.decode_sfixed64(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(buf=st.binary(max_size=100), pos=st.integers(max_value=200)) def test_decode_double(buf, pos): try: fixed.decode_double(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass # Varint exception tests @given(value=st.integers()) def test_encode_uvarint(value): try: varint.encode_uvarint(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(value=st.integers()) def test_encode_varint(value): try: varint.encode_varint(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(value=st.integers()) def test_encode_svarint(value): try: varint.encode_svarint(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(buf=st.binary(max_size=32), pos=st.integers(max_value=64)) def test_decode_uvarint(buf, pos): try: varint.decode_uvarint(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(buf=st.binary(max_size=32), pos=st.integers(max_value=64)) def test_decode_varint(buf, pos): try: varint.decode_varint(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(buf=st.binary(max_size=32), pos=st.integers(max_value=64)) def test_decode_svarint(buf, pos): try: varint.decode_svarint(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass # length_delim exception tests @given(value=st.binary()) def encode_bytes(value): try: length_delim.encode_bytes(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(value=st.binary(), pos=st.integers(max_value=2000)) def test_decode_bytes(value, pos): try: length_delim.decode_bytes(value, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(value=st.binary()) def test_encode_bytes_hex(value): try: length_delim.encode_bytes_hex(value) except BlackboxProtobufException as exc: assert not isinstance(exc, DecoderException) pass @given(buf=st.binary(), pos=st.integers(max_value=2000)) def test_decode_bytes_hex(buf, pos): try: length_delim.decode_bytes_hex(buf, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(value=st.binary(), pos=st.integers(max_value=2000)) def test_decode_string(value, pos): try: length_delim.decode_string(value, pos) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(buf=st.binary()) def test_decode_message(buf): try: length_delim.decode_message(buf, config.Config()) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass @given(buf=st.binary()) def test_decode_lendelim_message(buf): try: length_delim.decode_lendelim_message(buf, config.Config()) except BlackboxProtobufException as exc: assert not isinstance(exc, EncoderException) pass blackboxprotobuf-1.4.2/lib/tests/py_test/test_fixed.py000066400000000000000000000105271476504373500232160ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import math from hypothesis import given import hypothesis.strategies as st import strategies from blackboxprotobuf.lib.types import fixed # Inverse checks. Ensure a value encoded by bbp decodes to the same value @given(x=strategies.input_map["fixed32"]) def test_fixed32_inverse(x): encoded = fixed.encode_fixed32(x) decoded, pos = fixed.decode_fixed32(encoded, 0) assert pos == len(encoded) assert decoded == x @given(x=st.binary(min_size=4)) def test_fixed32_idem(x): try: value, pos = fixed.decode_fixed32(x, 0) except DecoderException: assume(True) return encoded = fixed.encode_fixed32(value) assert encoded == x[:pos] @given(x=strategies.input_map["sfixed32"]) def test_sfixed32_inverse(x): encoded = fixed.encode_sfixed32(x) decoded, pos = fixed.decode_sfixed32(encoded, 0) assert pos == len(encoded) assert decoded == x @given(x=st.binary(min_size=4)) def test_sfixed32_idem(x): try: value, pos = fixed.decode_sfixed32(x, 0) except DecoderException: assume(True) return encoded = fixed.encode_sfixed32(value) assert encoded == x[:pos] @given(x=strategies.input_map["fixed64"]) def test_fixed64_inverse(x): encoded = fixed.encode_fixed64(x) decoded, pos = fixed.decode_fixed64(encoded, 0) assert pos == len(encoded) assert decoded == x @given(x=st.binary(min_size=8)) def test_fixed64_idem(x): try: value, pos = fixed.decode_fixed64(x, 0) except DecoderException: assume(True) return encoded = fixed.encode_fixed64(value) assert encoded == x[:pos] @given(x=strategies.input_map["sfixed64"]) def test_sfixed64_inverse(x): encoded = fixed.encode_sfixed64(x) decoded, pos = fixed.decode_sfixed64(encoded, 0) assert pos == len(encoded) assert decoded == x @given(x=st.binary(min_size=8)) def test_sfixed64_idem(x): try: value, pos = fixed.decode_sfixed64(x, 0) except DecoderException: assume(True) return encoded = fixed.encode_sfixed64(value) assert encoded == x[:pos] @given(x=strategies.input_map["float"]) def test_float_inverse(x): encoded = fixed.encode_float(x) decoded, pos = fixed.decode_float(encoded, 0) assert pos == len(encoded) if math.isnan(x): assert math.isnan(decoded) else: assert decoded == x # Would be nice, but not a default type, so probably ok # Probably asking for trouble to have a float decode then encode the same # @given(x=st.binary(min_size=4)) # def test_float_idem(x): # try: # value, pos = fixed.decode_float(x, 0) # except DecoderException: # assume(True) # return # # encoded = fixed.encode_float(value) # assert encoded == x[:pos] @given(x=strategies.input_map["double"]) def test_double_inverse(x): encoded = fixed.encode_double(x) decoded, pos = fixed.decode_double(encoded, 0) assert pos == len(encoded) if math.isnan(x): assert math.isnan(decoded) else: assert decoded == x # @given(x=st.binary(min_size=8)) # def test_double_idem(x): # try: # value, pos = fixed.decode_double(x, 0) # except DecoderException: # assume(True) # return # # encoded = fixed.encode_double(value) # assert encoded == x[:pos] blackboxprotobuf-1.4.2/lib/tests/py_test/test_json.py000066400000000000000000000205761476504373500230750ustar00rootroot00000000000000"""Tests similar to the length_delim or protobuf tests, but make sure we can round trip through the JSON encode/decode """ # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from hypothesis import given, assume, note, example, reproduce_failure import hypothesis.strategies as st import strategies import json import six import binascii from blackboxprotobuf.lib.config import Config from blackboxprotobuf.lib.types import length_delim from blackboxprotobuf.lib.types import type_maps from blackboxprotobuf.lib.typedef import TypeDef from blackboxprotobuf.lib.payloads import grpc, gzip import blackboxprotobuf @given(x=strategies.gen_message()) def test_message_json_inverse(x): config = Config() typedef, message = x encoded = length_delim.encode_message(message, config, TypeDef.from_dict(typedef)) decoded_json, typedef_out = blackboxprotobuf.protobuf_to_json( encoded, config=config, message_type=typedef ) blackboxprotobuf.validate_typedef(typedef_out) encoded_json = blackboxprotobuf.protobuf_from_json( decoded_json, config=config, message_type=typedef_out ) assert not isinstance(encoded_json, list) decoded, typedef_out = blackboxprotobuf.decode_message( encoded_json, config=config, message_type=typedef ) blackboxprotobuf.validate_typedef(typedef_out) assert isinstance(encoded, bytearray) assert isinstance(decoded, dict) assert message == decoded @given(x=strategies.gen_message(), n=st.integers(min_value=2, max_value=10)) def test_multiple_encoding(x, n): config = Config() typedef, message = x encoded = length_delim.encode_message(message, config, TypeDef.from_dict(typedef)) bufs = [encoded] * n message_json, typedef_out = blackboxprotobuf.protobuf_to_json(bufs, typedef, config) messages = json.loads(message_json) assert isinstance(messages, list) assert len(messages) == n encoded2 = blackboxprotobuf.protobuf_from_json(message_json, typedef, config) assert isinstance(encoded2, list) assert len(encoded2) == n @given(x=strategies.gen_message(anon=True)) def test_anon_json_decode(x): """Create a new encoded message, the try to decode without a typedef into a json, from json back to binary and then finally decode the message back to the original format. Makes sure json decoding can handle any types and does not change the essage. """ config = Config() typedef, message = x encoded = blackboxprotobuf.encode_message( message, config=config, message_type=typedef ) decoded_json, typedef_out = blackboxprotobuf.protobuf_to_json( encoded, config=config ) blackboxprotobuf.validate_typedef(typedef_out) note("To Json Typedef: %r" % dict(typedef_out)) encoded_json = blackboxprotobuf.protobuf_from_json( decoded_json, config=config, message_type=typedef_out ) assert not isinstance(encoded_json, list) decoded, typedef_out = blackboxprotobuf.decode_message( encoded_json, config=config, message_type=typedef ) blackboxprotobuf.validate_typedef(typedef_out) note("Original message: %r" % message) note("Decoded JSON: %r" % decoded_json) note("Decoded message: %r" % decoded) note("Original typedef: %r" % typedef) note("Decoded typedef: %r" % typedef_out) def check_message(orig, orig_typedef, new, new_typedef): for field_number in set(orig.keys()) | set(new.keys()): # verify all fields are there assert field_number in orig assert field_number in orig_typedef assert field_number in new assert field_number in new_typedef orig_values = orig[field_number] new_values = new[field_number] orig_type = orig_typedef[field_number]["type"] new_type = new_typedef[field_number]["type"] note("Parsing field# %s" % field_number) note("orig_values: %r" % orig_values) note("new_values: %r" % new_values) note("orig_type: %s" % orig_type) note("new_type: %s" % new_type) # Fields might be lists. Just convert everything to a list if not isinstance(orig_values, list): orig_values = [orig_values] assert not isinstance(new_values, list) new_values = [new_values] assert isinstance(orig_values, list) assert isinstance(new_values, list) # if the types don't match, then try to convert them if new_type == "message" and orig_type in ["bytes", "string"]: # if the type is a message, we want to convert the orig type to a message # this isn't ideal, we'll be using the unintended type, but # best way to compare. Re-encoding a message to binary might # not keep the field order new_field_typedef = new_typedef[field_number]["message_typedef"] for i, orig_value in enumerate(orig_values): if orig_type == "bytes": ( orig_values[i], orig_field_typedef, _, _, ) = length_delim.decode_lendelim_message( length_delim.encode_bytes(orig_value), config, TypeDef.from_dict(new_field_typedef), ) orig_field_typedef = orig_field_typedef.to_dict() else: # string value ( orig_values[i], orig_field_typedef, _, _, ) = length_delim.decode_lendelim_message( length_delim.encode_string(orig_value), config, TypeDef.from_dict(new_field_typedef), ) orig_field_typedef = orig_field_typedef.to_dict() orig_typedef[field_number]["message_typedef"] = orig_field_typedef orig_type = "message" if new_type == "string" and orig_type == "bytes": # our bytes were accidently valid string new_type = "bytes" for i, new_value in enumerate(new_values): new_values[i], _ = length_delim.decode_bytes( length_delim.encode_string(new_value), 0 ) note("New values: %r" % new_values) # sort the lists with special handling for dicts orig_values.sort(key=lambda x: x if not isinstance(x, dict) else x.items()) new_values.sort(key=lambda x: x if not isinstance(x, dict) else x.items()) for orig_value, new_value in zip(orig_values, new_values): if orig_type == "message": check_message( orig_value, orig_typedef[field_number]["message_typedef"], new_value, new_typedef[field_number]["message_typedef"], ) else: assert orig_value == new_value check_message(message, typedef, decoded, typedef_out) # assert message == decoded blackboxprotobuf-1.4.2/lib/tests/py_test/test_length_delim.py000066400000000000000000000445471476504373500245630ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from hypothesis import given, assume, note, example, reproduce_failure import hypothesis.strategies as st import collections import strategies import six import copy import binascii from blackboxprotobuf.lib.config import Config from blackboxprotobuf.lib.types import length_delim from blackboxprotobuf.lib.types import type_maps from blackboxprotobuf.lib.typedef import TypeDef, FieldDef if six.PY2: string_types = (unicode, str) else: string_types = str # Inverse checks. Ensure a value encoded by bbp decodes to the same value @given(x=strategies.input_map["bytes"]) def test_bytes_inverse(x): encoded = length_delim.encode_bytes(x) decoded, pos = length_delim.decode_bytes(encoded, 0) assert isinstance(encoded, bytearray) assert isinstance(decoded, bytearray) assert pos == len(encoded) assert decoded == x # Inverse checks. Ensure a value encoded by bbp decodes to the same value @given(x=strategies.input_map["bytes"]) def test_bytes_guess_inverse(x): config = Config() # wrap the message in a new message so that it's a guess inside wrapper_typedef = {"1": {"type": "bytes"}} wrapper_message = {"1": x} encoded = length_delim.encode_lendelim_message( wrapper_message, config, TypeDef.from_dict(wrapper_typedef) ) value, typedef, _, pos = length_delim.decode_lendelim_message( encoded, config, TypeDef() ) typedef = typedef.to_dict() # would like to fail if it guesses wrong, but sometimes it might parse as a message assume(typedef["1"]["type"] == "bytes") assert isinstance(encoded, bytearray) assert isinstance(value["1"], bytearray) assert pos == len(encoded) assert value["1"] == x @given(x=strategies.input_map["bytes"].map(binascii.hexlify)) def test_bytes_hex_inverse(x): encoded = length_delim.encode_bytes_hex(x) decoded, pos = length_delim.decode_bytes_hex(encoded, 0) assert isinstance(encoded, bytearray) assert isinstance(decoded, (bytearray, bytes)) assert pos == len(encoded) assert decoded == x @given(x=strategies.input_map["string"]) def test_string_inverse(x): encoded = length_delim.encode_bytes(x) decoded, pos = length_delim.decode_string(encoded, 0) assert isinstance(encoded, bytearray) assert isinstance(decoded, string_types) assert pos == len(encoded) assert decoded == x @given(x=strategies.gen_message()) def test_message_inverse(x): config = Config() typedef, message = x encoded = length_delim.encode_lendelim_message( message, config, TypeDef.from_dict(typedef) ) decoded, typedef_out, _, pos = length_delim.decode_lendelim_message( encoded, config, TypeDef.from_dict(typedef), 0 ) typedef_out = typedef_out.to_dict() note(encoded) note(typedef) note(typedef_out) assert isinstance(encoded, bytearray) assert isinstance(decoded, dict) assert pos == len(encoded) assert message == decoded @given(x=strategies.gen_message(anon=True)) def test_anon_decode(x): config = Config() typedef, message = x encoded = length_delim.encode_lendelim_message( message, config, TypeDef.from_dict(typedef) ) decoded, typedef_out, _, pos = length_delim.decode_lendelim_message( encoded, config, TypeDef(), 0 ) typedef_out = typedef_out.to_dict() note("Original message: %r" % message) note("Decoded message: %r" % decoded) note("Original typedef: %r" % typedef) note("Decoded typedef: %r" % typedef_out) def check_message(orig, orig_typedef, new, new_typedef): for field_number in set(orig.keys()) | set(new.keys()): # Skip cases where we accidentally wind up with an alt-typedef assume("-" not in field_number) # verify all fields are there assert field_number in orig assert field_number in orig_typedef assert field_number in new assert field_number in new_typedef orig_values = orig[field_number] new_values = new[field_number] orig_type = orig_typedef[field_number]["type"] new_type = new_typedef[field_number]["type"] note("Parsing field# %s" % field_number) note("orig_values: %r" % orig_values) note("new_values: %r" % new_values) note("orig_type: %s" % orig_type) note("new_type: %s" % new_type) # Fields might be lists. Just convert everything to a list if not isinstance(orig_values, list): orig_values = [orig_values] assert not isinstance(new_values, list) new_values = [new_values] # if the types don't match, then try to convert them if new_type == "message" and orig_type in ["bytes", "string"]: # if the type is a message, we want to convert the orig type to a message # this isn't ideal, we'll be using the unintended type, but # best way to compare. Re-encoding a message to binary might # not keep the field order new_field_typedef = new_typedef[field_number]["message_typedef"] for i, orig_value in enumerate(orig_values): if orig_type == "bytes": ( orig_values[i], orig_field_typedef, _, _, ) = length_delim.decode_lendelim_message( length_delim.encode_bytes(orig_value), config, TypeDef.from_dict(new_field_typedef), ) orig_field_typedef = orig_field_typedef.to_dict() else: # string value ( orig_values[i], orig_field_typedef, _, _, ) = length_delim.decode_lendelim_message( length_delim.encode_string(orig_value), config, TypeDef.from_dict(new_field_typedef), ) orig_field_typedef = orig_field_typedef.to_dict() orig_typedef[field_number]["message_typedef"] = orig_field_typedef orig_type = "message" if new_type == "string" and orig_type == "bytes": # our bytes were accidently valid string new_type = "bytes" for i, new_value in enumerate(new_values): new_values[i], _ = length_delim.decode_bytes( length_delim.encode_string(new_value), 0 ) # sort the lists with special handling for dicts orig_values.sort(key=lambda x: x if not isinstance(x, dict) else x.items()) new_values.sort(key=lambda x: x if not isinstance(x, dict) else x.items()) for orig_value, new_value in zip(orig_values, new_values): if orig_type == "message": check_message( orig_value, orig_typedef[field_number]["message_typedef"], new_value, new_typedef[field_number]["message_typedef"], ) else: assert orig_value == new_value check_message(message, typedef, decoded, typedef_out) @given(x=strategies.gen_message()) @example(x=({"1": {"seen_repeated": True, "type": "string"}}, {"1": ["", "0"]})) @example( x=( { "1": {"seen_repeated": False, "type": "sfixed32"}, "2": {"seen_repeated": True, "type": "string"}, }, {"1": 0, "2": ["0", "00"]}, ) ) def test_message_guess_inverse(x): config = Config() type_def, message = x # wrap the message in a new message so that it's a guess inside wrapper_typedef = {"1": {"type": "message", "message_typedef": type_def}} wrapper_message = {"1": message} encoded = length_delim.encode_lendelim_message( wrapper_message, config, TypeDef.from_dict(wrapper_typedef) ) note("Encoded length %d" % len(encoded)) value, decoded_type, _, pos = length_delim.decode_lendelim_message( encoded, config, TypeDef() ) decoded_type = decoded_type.to_dict() note(value) assert decoded_type["1"]["type"] == "message" assert isinstance(encoded, bytearray) assert isinstance(value, dict) assert isinstance(value["1"], dict) assert pos == len(encoded) @given(bytes_in=st.binary()) def test_message_guess_bytes(bytes_in): # Test that a given byte array can be decoded anonymously then re-encoded to the same bytes config = Config() # embed it in a another message so we get proper type guessing wrapper_typedef = {"1": {"type": "bytes"}} wrapper_message = {"1": bytes_in} bytes_in = length_delim.encode_message( wrapper_message, config, TypeDef.from_dict(wrapper_typedef) ) decoded_message, guessed_typedef, field_order, pos = length_delim.decode_message( bytes_in, config, TypeDef() ) guessed_typedef = guessed_typedef.to_dict() assert pos == len(bytes_in) bytes_out = length_delim.encode_message( decoded_message, config, TypeDef.from_dict(guessed_typedef) ) assert bytes_in == bytes_out @given(x=strategies.gen_message(anon=True), rng=st.randoms()) def test_message_ordering(x, rng): # messages need to preserve field ordering when encoding then decoding # ordering technically shouldn't matter in a protobuf message, but if we # decode a non-protobuf message as a protobuf and then re-encode it to # bytes, it will scramble the bytes and violate the rule that decoding then # re-encoding shouldn't change the message config = Config() typedef, message = x # wrap the message in a new message so that it's a guess inside typedef = {"1": {"type": "message", "message_typedef": typedef}} message = {"1": message} # encode to bytes first message_bytes = length_delim.encode_message( message, config, TypeDef.from_dict(typedef) ) # now we have bytes that could be decoded as a message, we don't care what the original typedef is decoded_message, typedef, _, _ = length_delim.decode_message( message_bytes, config, TypeDef() ) typedef = typedef.to_dict() message_items = list(decoded_message["1"].items()) rng.shuffle(message_items) decoded_message["1"] = collections.OrderedDict(message_items) new_message_bytes = length_delim.encode_message( decoded_message, config, TypeDef.from_dict(typedef) ) assert message_bytes == new_message_bytes @given(x=strategies.input_map["packed_uint"]) def test_packed_uint_inverse(x): encoded = type_maps.ENCODERS["packed_uint"](x) decoded, pos = type_maps.DECODERS["packed_uint"](encoded, 0) assert isinstance(encoded, bytearray) assert pos == len(encoded) assert x == decoded @given(x=strategies.input_map["packed_int"]) def test_packed_int_inverse(x): encoded = type_maps.ENCODERS["packed_int"](x) decoded, pos = type_maps.DECODERS["packed_int"](encoded, 0) assert isinstance(encoded, bytearray) assert pos == len(encoded) assert x == decoded @given(x=strategies.input_map["packed_sint"]) def test_packed_sint_inverse(x): encoded = type_maps.ENCODERS["packed_sint"](x) decoded, pos = type_maps.DECODERS["packed_sint"](encoded, 0) assert isinstance(encoded, bytearray) assert pos == len(encoded) assert x == decoded @given(x=strategies.input_map["packed_fixed32"]) def test_packed_fixed32_inverse(x): encoded = type_maps.ENCODERS["packed_fixed32"](x) decoded, pos = type_maps.DECODERS["packed_fixed32"](encoded, 0) assert isinstance(encoded, bytearray) assert pos == len(encoded) assert x == decoded @given(x=strategies.input_map["packed_sfixed32"]) def test_packed_sfixed32_inverse(x): encoded = type_maps.ENCODERS["packed_sfixed32"](x) decoded, pos = type_maps.DECODERS["packed_sfixed32"](encoded, 0) assert isinstance(encoded, bytearray) assert pos == len(encoded) assert x == decoded @given(x=strategies.input_map["packed_float"]) def test_packed_float_inverse(x): encoded = type_maps.ENCODERS["packed_float"](x) decoded, pos = type_maps.DECODERS["packed_float"](encoded, 0) assert isinstance(encoded, bytearray) assert pos == len(encoded) assert x == decoded @given(x=strategies.input_map["packed_fixed64"]) def test_packed_fixed64_inverse(x): encoded = type_maps.ENCODERS["packed_fixed64"](x) decoded, pos = type_maps.DECODERS["packed_fixed64"](encoded, 0) assert isinstance(encoded, bytearray) assert pos == len(encoded) assert x == decoded @given(x=strategies.input_map["packed_sfixed64"]) def test_packed_sfixed64_inverse(x): encoded = type_maps.ENCODERS["packed_sfixed64"](x) decoded, pos = type_maps.DECODERS["packed_sfixed64"](encoded, 0) assert isinstance(encoded, bytearray) assert pos == len(encoded) assert x == decoded @given(x=strategies.input_map["packed_double"]) def test_packed_double_inverse(x): encoded = type_maps.ENCODERS["packed_double"](x) decoded, pos = type_maps.DECODERS["packed_double"](encoded, 0) assert isinstance(encoded, bytearray) assert pos == len(encoded) assert x == decoded def test_seen_repeated(): # Make sure seen_repeated gets set and perserved config = Config() message = {"1": [1, 2, 3], "2": [{"1": 1}, {"1": 1}]} typedef = { "1": {"type": "int"}, "2": {"type": "message", "message_typedef": {"1": {"type": "int"}}}, } # Make sure we set seen_repeated for lists with multiple items encoded = length_delim.encode_lendelim_message( message, config, TypeDef.from_dict(typedef) ) decoded, typedef_out, _, pos = length_delim.decode_lendelim_message( encoded, config, TypeDef.from_dict(typedef), 0 ) typedef_out = typedef_out.to_dict() assert "seen_repeated" in typedef_out["1"] assert typedef_out["1"]["seen_repeated"] assert "seen_repeated" in typedef_out["2"] assert typedef_out["2"]["seen_repeated"] message = {"1": 1, "2": {"1": 1}} encoded = length_delim.encode_lendelim_message( message, config, TypeDef.from_dict(typedef) ) decoded, typedef_out, _, pos = length_delim.decode_lendelim_message( encoded, config, TypeDef.from_dict(typedef), 0 ) typedef_out = typedef_out.to_dict() # Make sure we don't set seen_repeated for single assert "seen_repeated" not in typedef_out["1"] assert "seen_repeated" not in typedef_out["2"] typedef["1"]["seen_repeated"] = True typedef["2"]["seen_repeated"] = True decoded, typedef_out, _, pos = length_delim.decode_lendelim_message( encoded, config, TypeDef.from_dict(typedef), 0 ) typedef_out = typedef_out.to_dict() # Make sure we preserve seen_repeated and output as a list assert "seen_repeated" in typedef_out["1"] assert typedef_out["1"]["seen_repeated"] # Make sure our output is a list, even though it only has one list assert isinstance(decoded["1"], list) assert "seen_repeated" in typedef_out["2"] assert typedef_out["2"]["seen_repeated"] # Make sure our output is a list, even though it only has one list assert isinstance(decoded["2"], list) def test_immutable_typedef(): # we want to ensure that the original typedef is never modified by a decode operation config = Config() typedef0 = { "1": {"type": "int"}, "2": { "type": "message", "message_typedef": {"1": {"type": "int"}}, "alt_typedefs": { "2": "bytes", "3": {"1": {"type": "fixed64"}}, }, }, } typedef0_deepcopy = copy.deepcopy(typedef0) message0 = { "1": 1, "2": {"1": 1}, } data0 = length_delim.encode_lendelim_message( message0, config, TypeDef.from_dict(typedef0) ) typedef1 = { "1": {"type": "int"}, "2": {"type": "string"}, } message1 = { "1": 5, "2": "Test123", } data1 = length_delim.encode_lendelim_message( message1, config, TypeDef.from_dict(typedef1) ) length_delim.decode_lendelim_message(data1, config, TypeDef.from_dict(typedef0)) assert typedef0 == typedef0_deepcopy typedef2 = { "1": {"type": "int"}, "2": { "type": "message", "message_typedef": {"1": {"type": "int"}, "2": {"type": "int"}}, }, "3": {"type": "int"}, } message2 = { "1": 7, "2": { "1": 1, "2": 3, }, "3": 8, } data2 = length_delim.encode_lendelim_message( message2, config, TypeDef.from_dict(typedef2) ) length_delim.decode_lendelim_message(data2, config, TypeDef.from_dict(typedef0)) assert typedef0 == typedef0_deepcopy @given(x=st.binary()) def test_bytes_fallback(x): # Make sure we fallback to bytes, event if our default_binary_type fails config = Config() config.default_binary_type = "string" encoded = length_delim.encode_bytes(x) decoded, pos = length_delim._try_decode_lendelim_fields( [encoded], FieldDef(1), config, [] ) blackboxprotobuf-1.4.2/lib/tests/py_test/test_payloads.py000066400000000000000000000100751476504373500237310ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from hypothesis import assume, given, reproduce_failure, example import hypothesis.strategies as st import strategies import pytest from blackboxprotobuf.lib import payloads from blackboxprotobuf.lib.payloads import grpc, gzip from blackboxprotobuf.lib.exceptions import BlackboxProtobufException def test_grpc(): message = bytearray([0x00, 0x00, 0x00, 0x00, 0x01, 0xAA]) data, encoding = grpc.decode_grpc(message) assert data == bytearray([0xAA]) assert encoding == "grpc" # Compression flag with pytest.raises(BlackboxProtobufException): message = bytearray([0x01, 0x00, 0x00, 0x00, 0x01, 0xAA]) data = grpc.decode_grpc(message) # Unknown flag with pytest.raises(BlackboxProtobufException): message = bytearray([0x11, 0x00, 0x00, 0x00, 0x01, 0xAA]) data = grpc.decode_grpc(message) # Incorrect length with pytest.raises(BlackboxProtobufException): message = bytearray([0x00, 0x00, 0x01, 0x00, 0x01, 0xAA]) data = grpc.decode_grpc(message) # Incorrect length with pytest.raises(BlackboxProtobufException): message = bytearray([0x00, 0x00, 0x00, 0x00, 0x01, 0xAA, 0xBB]) data = grpc.decode_grpc(message) # Empty message = bytearray([0x00, 0x00, 0x00, 0x00, 0x00]) data, encoding = grpc.decode_grpc(message) assert len(data) == 0 assert encoding == "grpc" @given(payloads=st.lists(st.binary(), min_size=2)) def test_grpc_multiple(payloads): # Test grpc encoding with multiple payloads # Manually encode multiple grpc payloads and string them together encoded = b"" for payload in payloads: encoded += grpc.encode_grpc(payload) assert grpc.is_grpc(encoded) # Make sure we can decode bytes with multiple grpc decoded, encoding = grpc.decode_grpc(encoded) assert isinstance(decoded, list) assert len(decoded) == len(payloads) for x, y in zip(decoded, payloads): assert x == y # Make sure we can encode to the same bytes encoded2 = grpc.encode_grpc(decoded) assert encoded == encoded2 @given(data=st.binary()) def test_grpc_inverse(data): encoding = "grpc" encoded = grpc.encode_grpc(data) decoded, encoding_out = grpc.decode_grpc(encoded) assert data == decoded assert encoding == encoding_out @given(data=st.binary()) def test_gzip_inverse(data): encoded = gzip.encode_gzip(data) decoded, encoding_out = gzip.decode_gzip(encoded) assert data == decoded assert "gzip" == encoding_out @given(data=st.binary(), alg=st.sampled_from(["grpc", "gzip", "none"])) def test_find_payload_inverse(data, alg): encoded = payloads.encode_payload(data, alg) decoders = payloads.find_decoders(encoded) valid_decoders = {} for decoder in decoders: try: decoded, decoder_alg = decoder(encoded) valid_decoders[decoder_alg] = decoded except: pass assert "none" in valid_decoders assert alg in valid_decoders assert valid_decoders[alg] == data blackboxprotobuf-1.4.2/lib/tests/py_test/test_perf.py000066400000000000000000000030141476504373500230440ustar00rootroot00000000000000import pytest import blackboxprotobuf @pytest.mark.skip() def test_wide(): typedef = {"1": {"type": "int"}} message = {"1": [1] * 10000000} encoded = blackboxprotobuf.lib.encode_message(message, typedef) decoded, _ = blackboxprotobuf.lib.decode_message(encoded, typedef) @pytest.mark.skip() def test_deep(): config = blackboxprotobuf.lib.config.Config() typedef = { "1": {"type": "message", "message_type_name": "test"}, "2": {"type": "int"}, } config.known_types["test"] = typedef target_depth = 100 message = {} last_layer = message while target_depth: new_layer = {"2": 1} last_layer["1"] = new_layer last_layer = new_layer target_depth -= 1 encoded = blackboxprotobuf.lib.encode_message(message, typedef, config) decoded, _ = blackboxprotobuf.lib.decode_message(encoded, typedef, config) @pytest.mark.skip() def test_large_multilayer(): config = blackboxprotobuf.lib.config.Config() typedef = { "1": {"type": "message", "message_type_name": "test"}, "2": {"type": "int"}, } config.known_types["test"] = typedef target_depth = 10 message = {} last_layer = message while target_depth: new_layer = {"2": [1] * 10000} last_layer["1"] = [new_layer] * 2 last_layer = new_layer target_depth -= 1 encoded = blackboxprotobuf.lib.encode_message(message, typedef, config) decoded, _ = blackboxprotobuf.lib.decode_message(encoded, typedef, config) blackboxprotobuf-1.4.2/lib/tests/py_test/test_protobuf.py000066400000000000000000000217311476504373500237560ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from hypothesis import given, example, note import hypothesis.strategies as st import hypothesis import strategies import warnings import base64 import json import six import blackboxprotobuf warnings.filterwarnings( "ignore", "Call to deprecated create function.*", ) try: import Test_pb2 except: import os os.system( "cd tests/payloads; protoc --python_out ../py_test/ Test.proto; cd ../../" ) import Test_pb2 testMessage_typedef = { "1": {"type": "double", "name": six.u("testDouble")}, "2": {"type": "float", "name": six.u("testFloat")}, # "4": {"type": "int", "name": "testInt32"}, "8": {"type": "int", "name": six.u("testInt64")}, # "16": {"type": "uint", "name": "testUInt32"}, "32": {"type": "uint", "name": six.u("testUInt64")}, # "64": {"type": "sint", "name": "testSInt32"}, "128": {"type": "sint", "name": six.u("testSInt64")}, "256": {"type": "fixed32", "name": six.u("testFixed32")}, "512": {"type": "fixed64", "name": six.u("testFixed64")}, "1024": {"type": "sfixed32", "name": six.u("testSFixed32")}, "2048": {"type": "sfixed64", "name": six.u("testSFixed64")}, # "4096": {"type": "int", "name": "testBool"}, "8192": {"type": "string", "name": six.u("testString")}, "16384": {"type": "bytes", "name": six.u("testBytes")}, # "32768": {"type": "message", "name": "testEmbed", # "message_typedef": { # "3": {"type": "double", "name": "embedDouble"}, # "2": {"type": "bytes", "name": "embedString"}} # }, # "65536": {"type": "packed_int", "name": "testRepeatedInt32"} } # Test decoding from blackboxprotobuf @given(x=strategies.gen_message_data(testMessage_typedef)) def test_decode(x): message = Test_pb2.TestMessage() for key, value in x.items(): setattr(message, key, value) encoded = message.SerializeToString() decoded, typedef = blackboxprotobuf.decode_message(encoded, testMessage_typedef) blackboxprotobuf.validate_typedef(typedef) hypothesis.note("Decoded: %r" % decoded) for key in decoded.keys(): assert x[key] == decoded[key] # Test encoding with blackboxprotobuf @given(x=strategies.gen_message_data(testMessage_typedef)) def test_encode(x): encoded = blackboxprotobuf.encode_message(x, testMessage_typedef) message = Test_pb2.TestMessage() message.ParseFromString(encoded) for key in x.keys(): assert getattr(message, key) == x[key] # Try to modify a random key with blackbox and re-encode # TODO: In the future do more random modifications, like swap the whole value @given( x=strategies.gen_message_data(testMessage_typedef), modify_num=st.sampled_from(sorted(testMessage_typedef.keys())), ) def test_modify(x, modify_num): modify_key = testMessage_typedef[modify_num]["name"] message = Test_pb2.TestMessage() for key, value in x.items(): setattr(message, key, value) encoded = message.SerializeToString() decoded, typedef = blackboxprotobuf.decode_message(encoded, testMessage_typedef) blackboxprotobuf.validate_typedef(typedef) # eliminate any cases where protobuf defaults out a field hypothesis.assume(modify_key in decoded) if isinstance(decoded[modify_key], six.text_type): mod_func = lambda x: six.u("test") elif isinstance(decoded[modify_key], bytes): mod_func = lambda x: b"test" elif isinstance(decoded[modify_key], six.integer_types): mod_func = lambda x: 10 elif isinstance(decoded[modify_key], float): mod_func = lambda x: 10 else: hypothesis.note( "Failed to modify key: %s (%r)" % (modify_key, type(decoded[modify_key])) ) assert False decoded[modify_key] = mod_func(decoded[modify_key]) x[modify_key] = mod_func(x[modify_key]) encoded = blackboxprotobuf.encode_message(decoded, testMessage_typedef) message = Test_pb2.TestMessage() message.ParseFromString(encoded) for key in decoded.keys(): assert getattr(message, key) == x[key] ## Second copies of the above methods that use the protobuf to/from json functions @given(x=strategies.gen_message_data(testMessage_typedef)) @example(x={"testBytes": b"test123"}) @example(x={"testBytes": b"\x80"}) def test_decode_json(x): # Test with JSON payload message = Test_pb2.TestMessage() for key, value in x.items(): setattr(message, key, value) encoded = message.SerializeToString() decoded_json, typedef_json = blackboxprotobuf.protobuf_to_json( encoded, testMessage_typedef ) blackboxprotobuf.validate_typedef(typedef_json) hypothesis.note("Encoded JSON:") hypothesis.note(decoded_json) decoded = json.loads(decoded_json) hypothesis.note("Original value:") hypothesis.note(x) hypothesis.note("Decoded valuec:") hypothesis.note(decoded) for key in decoded.keys(): if key == "testBytes": decoded[key] = six.ensure_binary(decoded[key], encoding="latin1") assert x[key] == decoded[key] @given(x=strategies.gen_message_data(testMessage_typedef)) @example(x={"testBytes": b"\x80"}) def test_encode_json(x): # Test with JSON payload if "testBytes" in x: x["testBytes"] = x["testBytes"].decode("latin1") json_str = json.dumps(x) hypothesis.note("JSON Str Input:") hypothesis.note(json_str) hypothesis.note(json.loads(json_str)) encoded = blackboxprotobuf.protobuf_from_json(json_str, testMessage_typedef) assert not isinstance(encoded, list) hypothesis.note("BBP decoding:") test_decode, _ = blackboxprotobuf.decode_message(encoded, testMessage_typedef) hypothesis.note(test_decode) message = Test_pb2.TestMessage() message.ParseFromString(encoded) hypothesis.note("Message:") hypothesis.note(message) for key in x.keys(): hypothesis.note("Message value") hypothesis.note(type(getattr(message, key))) hypothesis.note("Original value") hypothesis.note(type(x[key])) if key == "testBytes": x[key] = six.ensure_binary(x[key], encoding="latin1") assert getattr(message, key) == x[key] @given( x=strategies.gen_message_data(testMessage_typedef), modify_num=st.sampled_from(sorted(testMessage_typedef.keys())), ) def test_modify_json(x, modify_num): modify_key = testMessage_typedef[modify_num]["name"] message = Test_pb2.TestMessage() for key, value in x.items(): setattr(message, key, value) encoded = message.SerializeToString() decoded_json, typedef = blackboxprotobuf.protobuf_to_json( encoded, testMessage_typedef ) blackboxprotobuf.validate_typedef(typedef) decoded = json.loads(decoded_json) # eliminate any cases where protobuf defaults out a field hypothesis.assume(modify_key in decoded) if isinstance(decoded[modify_key], six.text_type): mod_func = lambda x: six.u("test") elif isinstance(decoded[modify_key], bytes): mod_func = lambda x: b"test" elif isinstance(decoded[modify_key], six.integer_types): mod_func = lambda x: 10 elif isinstance(decoded[modify_key], float): mod_func = lambda x: 10 else: hypothesis.note( "Failed to modify key: %s (%r)" % (modify_key, type(decoded[modify_key])) ) assert False decoded[modify_key] = mod_func(decoded[modify_key]) x[modify_key] = mod_func(x[modify_key]) encoded = blackboxprotobuf.protobuf_from_json( json.dumps(decoded), testMessage_typedef ) assert not isinstance(encoded, list) message = Test_pb2.TestMessage() message.ParseFromString(encoded) for key in decoded.keys(): hypothesis.note("Message value:") hypothesis.note(type(getattr(message, key))) hypothesis.note("Orig value:") hypothesis.note((x[key])) if key == "testBytes": x[key] = six.ensure_binary(x[key], encoding="latin1") assert getattr(message, key) == x[key] blackboxprotobuf-1.4.2/lib/tests/py_test/test_protofile.py000066400000000000000000000301001476504373500241070ustar00rootroot00000000000000# Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import sys import six import math import glob import shutil import base64 import struct import pytest import logging import tempfile import subprocess import hypothesis from hypothesis import given, assume, note, settings, HealthCheck import hypothesis.strategies as st import google.protobuf.json_format import strategies import blackboxprotobuf.lib import blackboxprotobuf.lib.protofile as protofile from blackboxprotobuf.lib.types import length_delim from blackboxprotobuf.lib.config import Config from blackboxprotobuf.lib.typedef import TypeDef to_suppress = [] if six.PY3: to_suppress = (HealthCheck.function_scoped_fixture,) @given( typedef=strategies.message_typedef_gen(named_fields=False), name=st.from_regex(protofile.NAME_REGEX), ) @settings(suppress_health_check=to_suppress) def test_proto_export(tmp_path, typedef, name): """Check to make sure our generated protofiles don't throw an error""" with tempfile.NamedTemporaryFile( mode="w", dir=str(tmp_path), suffix=".proto", delete=True ) as outfile: typedef_map = {name: typedef} note(typedef_map) # Trying exporting as string first protofile.export_proto(typedef_map) protofile.export_proto(typedef_map, output_file=outfile) py_out = str(tmp_path / "py_out") if os.path.exists(py_out): shutil.rmtree(py_out) os.mkdir(py_out) outfile.flush() subprocess.check_call( "/usr/bin/protoc --python_out ./py_out %s" % os.path.basename(outfile.name), shell=True, cwd=str(tmp_path), ) @given( x=strategies.gen_message(named_fields=False), name=st.from_regex(protofile.NAME_REGEX), ) @settings(suppress_health_check=to_suppress) def test_proto_export_inverse(tmp_path, x, name): """Generate a proto file and try to re-import it. This does not cover all possible proto files we want to try importing""" config = Config() typedef, message = x name = six.ensure_text(name) with tempfile.NamedTemporaryFile( mode="r+", dir=str(tmp_path), suffix=".proto", delete=True ) as outfile: typedef_map = {name: typedef} protofile.export_proto(typedef_map, output_file=outfile) outfile.flush() outfile.seek(0) new_typedef_map = protofile.import_proto(config, input_file=outfile) config.known_types.update(new_typedef_map) # validate for name, typedef in new_typedef_map.items(): blackboxprotobuf.validate_typedef(typedef, config=config) def _check_field_types(typedef1, typedef2): for field_num in typedef1.keys(): # make sure we don't drop keys assert field_num in typedef2 assert typedef1[field_num]["type"] == typedef2[field_num]["type"] if typedef1[field_num]["type"] == "message": message_typedef1 = None message_typedef2 = None if "message_typedef" in typedef1[field_num]: message_typedef1 = typedef1[field_num]["message_typedef"] elif "message_type_name" in typedef1[field_num]: assert typedef1[field_num]["message_type_name"] in typedef_map message_typedef1 = typedef_map[ typedef1[field_num]["message_type_name"] ] if "message_typedef" in typedef2[field_num]: message_typedef2 = typedef2[field_num]["message_typedef"] elif "message_type_name" in typedef2[field_num]: assert ( typedef2[field_num]["message_type_name"] in new_typedef_map ) message_typedef2 = new_typedef_map[ typedef2[field_num]["message_type_name"] ] _check_field_types(message_typedef1, message_typedef2) note(typedef_map) note(new_typedef_map) for name, typedef in typedef_map.items(): _check_field_types(typedef, new_typedef_map[name]) note(new_typedef_map[name]) # try to actually encode a message with the typedef encode_forward = length_delim.encode_message( message, config, TypeDef.from_dict(typedef_map[name]) ) config.known_types = new_typedef_map encode_backward = length_delim.encode_message( message, config, TypeDef.from_dict(new_typedef_map[name]) ) decode_forward, _, _, _ = length_delim.decode_message( encode_forward, config, TypeDef.from_dict(new_typedef_map[name]) ) decode_backward, _, _, _ = length_delim.decode_message( encode_backward, config, TypeDef.from_dict(typedef_map[name]) ) @pytest.mark.filterwarnings("ignore:Call to deprecated create function.*") def test_proto_import_examples(): config = Config() # try importing all the examples pulled from protobuf repo protofiles = glob.glob("tests/deps/protobuf/src/google/protobuf/*.proto") # These files have some mechanism we don't support, mostly imports unsupported_files = { "tests/deps/protobuf/src/google/protobuf/api.proto", # import "tests/deps/protobuf/src/google/protobuf/unittest_optimize_for.proto", # import "tests/deps/protobuf/src/google/protobuf/type.proto", # import "tests/deps/protobuf/src/google/protobuf/unittest_lite_imports_nonlite.proto", # import "tests/deps/protobuf/src/google/protobuf/unittest_lite.proto", # group type not supported "tests/deps/protobuf/src/google/protobuf/unittest_embed_optimize_for.proto", # import "tests/deps/protobuf/src/google/protobuf/unittest.proto", # group "tests/deps/protobuf/src/google/protobuf/unittest_lazy_dependencies.proto", # import } assert len(protofiles) != 0 for target_file in protofiles: if target_file in unsupported_files: print("Skipping file: %s" % target_file) continue print("Testing file: %s" % target_file) typedef_map_out = protofile.import_proto(config, input_filename=target_file) config.known_types = typedef_map_out for name, typedef in typedef_map_out.items(): logging.debug("known messages: %s" % config.known_types) blackboxprotobuf.lib.validate_typedef(typedef, config=config) @given( x=strategies.gen_message(named_fields=False), name=st.from_regex(protofile.NAME_REGEX), ) @settings(suppress_health_check=to_suppress) @pytest.mark.filterwarnings("ignore:Call to deprecated create function.*") def test_proto_decode(tmp_path, x, name): config = Config() typedef, message = x """ Export to protobuf and try to decoe a message we encodedd with it """ with tempfile.NamedTemporaryFile( mode="w", dir=str(tmp_path), suffix=".proto", delete=True ) as outfile: typedef_map = {name: typedef} encoded_message = length_delim.encode_message( message, config, TypeDef.from_dict(typedef) ) note(typedef_map) basename = os.path.basename(outfile.name) # Export the protobuf file and compile it protofile.export_proto(typedef_map, output_file=outfile, package=basename[:-6]) py_out = str(tmp_path / "py_out") if os.path.exists(py_out): shutil.rmtree(py_out) os.mkdir(py_out) outfile.flush() subprocess.check_call( "/usr/bin/protoc --python_out ./py_out %s" % basename, shell=True, cwd=str(tmp_path), ) # Try to import the file sys.path.insert(0, str(tmp_path) + "/py_out/") # Trim off .proto try: proto_module = __import__(basename[:-6] + "_pb2") del sys.path[0] except SyntaxError: logging.debug("Caught syntax error in protoc import") return message_class = getattr(proto_module, name) note(encoded_message) my_message = message_class() my_message.ParseFromString(encoded_message) decoded_message = google.protobuf.json_format.MessageToDict( my_message, including_default_value_fields=True ) note(message) note(decoded_message) note( google.protobuf.json_format.MessageToJson( my_message, including_default_value_fields=True ) ) def _check_field_match(orig_value, new_value): note(type(new_value)) note(type(orig_value)) if isinstance(orig_value, six.integer_types) and isinstance(new_value, str): assert str(orig_value) == new_value elif isinstance(orig_value, bytes): assert orig_value == base64.b64decode(new_value) elif isinstance(new_value, dict): _check_message_match(orig_value, new_value) elif isinstance(orig_value, float): # normalize floats if isinstance(new_value, str): if "Infinity" in new_value: assert math.isinf(orig_value) else: assert new_value == "NaN" assert math.isnan(new_value) else: # pack and unpack floats to try and normalize them try: orig_value_packed = struct.pack("=5.0 hypothesis==4.57.1 blackboxprotobuf-1.4.2/lib/tests/run_decoder.py000066400000000000000000000025371476504373500216640ustar00rootroot00000000000000#!/usr/bin/env python # Copyright (c) 2018-2024 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import sys sys.path.insert(0, "../") import blackboxprotobuf as bbp typedef = {} # Take a protobuf binary from stdin and decode it to JSON protobuf = sys.stdin.read() json, typedef = bbp.protobuf_to_json(protobuf, typedef) print(json) print(typedef) blackboxprotobuf-1.4.2/mitmproxy/000077500000000000000000000000001476504373500171525ustar00rootroot00000000000000blackboxprotobuf-1.4.2/mitmproxy/README.md000066400000000000000000000130311476504373500204270ustar00rootroot00000000000000# Blackbox Protobuf Mitmproxy Addon ## Description This directory contains an addon for mitmproxy () which uses the Blackbox Protobuf library to decode and edit proxied protobuf messages. ## Installation 1. Clone the Blackboxprotobuf repository: ``` git clone https://github.com/nccgroup/blackboxprotobuf.git ``` 2. Update the submodules which contain dependencies: ``` cd blackboxprotobuf/ git submodule update --init ``` 3. Run mitmproxy with the addon: ``` mitmproxy -s mitmproxy/bbpb.py ``` ### Alternative Installation 1. Install the `bbpb` python package in the same context as `mitmproxy`. ``` pip install bbpb ``` If `mitmproxy` is installed within a virtual environment, then the `bbpb` package needs to be install within that virtual environment. 2. Download the `bbpb.py` from 3. Run mitmproxy with the addon script: ``` mitmproxy -s mitmproxy/bbpb.py ``` ## Usage ### Passive Decoding The Blackbox Protobuf addon provides a content view to automatically decode protobuf and gRPC messages and websockets displayed in mitmproxy. The content view will use saved types if available (eg. to name fields or change types), but will default to an anonymous decoding. The content view should also work in mitmweb, but commands for editing messages and types will not be available. ### Persisting Type Data The addon will remember edited types and the associated endpoints, but this data will be lost if mitmproxy is shut down or the addon is reloaded. The `bbpb_project_file` option in the mitmproxy configuration will cause the addon to load types from the provided file and then automatically write back any changes. If you have an extensive number of options, it's recommended to back up the project file regularly to ensure data isn't overwritten. The types can also be manually saved or loaded using the `:bbpb.project.save` and `:bbpb.project.load` commands. ### Commands Message and type editing functionality is provided via mitmproxy commands. The commands operate on the currently selected flow. Most commands take a `flow_part` argument, which indicates where the desired protobuf payload is in the message. Currently supported flow parts are: * `request-body` * `response-body` * `websocket` (for `bbpb.edit`, not supported with `bbpb.edit_type`) * `websocket-request` (for `bbpb.edit_type`, not supported for `bbpb.edit`) * `websocket-response` (for `bbpb.edit_type`, not supported for `bbpb.edit`) Command arguments support tab completion. #### `:bbpb.edit` The `:bbpb.edit` command will open a text editor with the protobuf decoded to JSON, similar to editing other HTTP messages in mitmproxy. The edited JSON payload will be re-encoded to protobuf to be replayed or resumed. The `request-body` and `response-body` flow parts will edit protobuf messages in the HTTP body. The `websocket` flow parts will edit the last message in the websocket flow. The addon does not support protobuf payloads elsewhere in the HTTP message at the moment, though you should be able to customize the addon for a particular use case. #### `:bbpb.edit_type` The `:bbpb.edit_type` command will open a text editor to edit the type definition used for the protobuf data. This can be used to add field names or change a field's type. If the endpoint is assigned a named type, then the named type is edited and the changes will be applied to all other endpoints using the same named type. If the type is not named, then the edited type definition is remembered just for that endpoint. #### `:bbpb.new_type` The `:bbpb.new_type` command will attach a name to the current type and store it in `known_types`. A named type can then be applied to multiple endpoints and you can switch between named types at-will without losing the type definition. #### `:bbpb.apply_type` The `:bbpb.apply_type` command will apply a previously saved named type to the endpoint/protobuf payload. Applying the type name of `(clear)` will remove the saved type for this endpoint, causing the decoding to be reset. Warning: If the current endpoint has an edited type which is not named, the edited type definition will be lost when a named type is applied. If the current endpoint is associated with another named type, the named type will be safe and can be reapplied later. #### `:bbpb.del_type` The `:bbpb.del_type` command can be used to remove a named type. Any endpoints associated with that named type will also be reset. #### `:bbpb.project.save` and `:bbpb.project.load` These commands will manually save/load a one-off Blackbox Protobuf project JSON file. This file contains all edited type definitions and all endpoint to type definition mappings. ## gRPC Note This addon has been tested with a python gRPC and client and is able to intercept messages by default. However, in the past mitmproxy has had some issues with intercepting gRPC connections from certain applications, and would register connections as PRI methods, or fail to intercept the request. The issue appears to be that some gRPC implementations use a unique `grpc-exp` ALPN value which proxies do not know how to handle. One work around that has worked in the past is cloning mitmproxy and changing if conditions which check for `h2` to also work for `grpc-exp`. Another potential solution is the addon modifications described at , but this has not been tested with the Blackbox Protobuf addon yet. blackboxprotobuf-1.4.2/mitmproxy/bbpb.py000066400000000000000000000605141476504373500204370ustar00rootroot00000000000000# Copyright (c) 2018-2023 NCC Group Plc # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import inspect import os import sys import logging try: import blackboxprotobuf except ModuleNotFoundError: # two abspath because dirname gives an empty string if we run just bbpb.py _BASE_DIR = os.path.abspath( os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) + "/.." ) sys.path.insert(0, _BASE_DIR + "/lib/") sys.path.insert(0, _BASE_DIR + "/mitmproxy/") sys.path.insert(0, _BASE_DIR + "/mitmproxy/deps/six/") import blackboxprotobuf from blackboxprotobuf.lib import payloads from blackboxprotobuf.lib.exceptions import BlackboxProtobufException import json from collections.abc import Sequence from typing import TypeVar, Callable, Optional from mitmproxy import ( command, contentviews, ctx, http, flow, types, exceptions, websocket, ) from mitmproxy.tools.console import overlay, signals, keymap class BlackboxProtobufAddon: def __init__(self): self.view = BlackboxProtobufView(self) self.bbpb_config = blackboxprotobuf.lib.config.Config() self.typedef_lookup = {} self.project_file = None def load(self, loader): contentviews.add(self.view) loader.add_option( "bbpb_project_file", typespec=str, default="", help="Persist known_types and typedef mappings to a project file. File will be written to automatically, so consider keeping a semi-regular backup", ) def _load_project_file(self, project_file: str | None = None): self.project_file = ctx.options.bbpb_project_file if not project_file and not self.project_file: return if not project_file: project_file = self.project_file if not self.project_file: return logging.info("Loading project data from file") try: with open(project_file, "r") as f: project_data = json.load(f) # We update the existing typedef_lookup and bbpb_confg in case we have existing data self.typedef_lookup.update(project_data["typedef_lookup"]) self.bbpb_config.known_types.update(project_data["known_types"]) except FileNotFoundError: # We haven't written anything, so the file might not exist yet pass self._refresh_view() def _save_project_file(self, project_file: str | None = None): if not project_file and not self.project_file: return if not project_file: project_file = self.project_file logging.info("Writing project data to file") data = { "typedef_lookup": self.typedef_lookup, "known_types": self.bbpb_config.known_types, } with open(project_file, "w") as f: json.dump(data, f, indent=2) def configure(self, updates: set[str]): if "bbpb_project_file" in updates: self._load_project_file() self._save_project_file() def done(self): contentviews.remove(self.view) @command.command("bbpb.edit") @command.argument("flow_part", type=types.Choice("bbpb.options.edit_part")) def bbpb_edit(self, flow_part: str) -> None: flow = ctx.master.view.focus.flow if flow_part == "request-body": message = flow.request elif flow_part == "response-body": if not flow.response: raise exceptions.CommandError( f"Flow part is response-body, but flow has no response" ) message = flow.response elif flow_part == "websocket": # Edit the last websocket message = flow.websocket.messages[-1] else: raise exceptions.CommandError(f"Got unknown flow_part: {flow_part}") message_hash = _message_hash(message.content, message, flow) typedef = self.typedef_lookup.get(message_hash) message_json, typedef_out, encoding_alg = _decode_protobuf( message.content, typedef, self.bbpb_config ) def message_callback(edited_json: str) -> None: protobuf_data = blackboxprotobuf.protobuf_from_json( edited_json, typedef_out ) data = payloads.encode_payload(protobuf_data, encoding_alg) message.content = bytes(data) self._refresh_view() _spawn_validating_editor(message_json, message_callback) @command.command("bbpb.edit_type") @command.argument("flow_part", type=types.Choice("bbpb.options.edit_type_part")) def bbpb_edit_type(self, flow_part: str) -> None: flow = ctx.master.view.focus.flow typedef, message_hash = self._resolve_type(flow_part) typedef_json = json.dumps(typedef, indent=2) def typedef_callback(typedef_json: str) -> None: new_typedef = json.loads(typedef_json) blackboxprotobuf.validate_typedef( new_typedef, typedef ) # Validate against old typedef blackboxprotobuf.lib.api._strip_typedef_annotations(new_typedef) known_type = self.typedef_lookup.get(message_hash) if isinstance(known_type, str): # This is a named typedef, edit the known typedef instead of the saved value self.bbpb_config.known_types[known_type] = new_typedef self._save_project_file() else: # Trusting validate_typedef and not going to try to use the typedef to decode again or reencode self.typedef_lookup[message_hash] = new_typedef self._save_project_file() self._refresh_view() _spawn_validating_editor(typedef_json, typedef_callback) @command.command("bbpb.apply_type") @command.argument("flow_part", type=types.Choice("bbpb.options.edit_type_part")) @command.argument("typename", type=types.Choice("bbpb.options.known_types")) def bbpb_apply_type(self, flow_part: str, typename: str) -> None: flow = ctx.master.view.focus.flow if typename not in self.bbpb_config.known_types and typename != "(clear)": raise exceptions.CommandError(f"Type {typename} is not a know type") flow = ctx.master.view.focus.flow if not flow: raise exceptions.CommandError("No flow selected.") if flow_part.startswith("request") or flow_part.startswith("response"): if flow_part == "request-body": message = flow.request elif flow_part == "response-body": if not flow.response: raise exceptions.CommandError( f"Flow part is response-body, but flow has no response" ) message = flow.response message_hash = _message_hash(message.content, message, flow) if typename == "(clear)": logging.info("popping message hash") self.typedef_lookup.pop(message_hash, None) self._save_project_file() self._refresh_view() return # Validate that we can decode the message with our new type try: _decode_protobuf( message.content, typename, self.bbpb_config, fallback=False ) except BlackboxProtobufException as ex: raise exceptions.CommandError( f"Error applying type name {typename} to part {flow_part}: {ex}" ) elif flow_part.startswith("websocket"): # Websockets don't have a single typedef to edit # Instead, we are going to build a typedef based on all the messages if flow_part == "websocket-request": if not flow.websocket: raise exceptions.CommandError( f"Flow part is websocket-request, but flow is not a websocket" ) messages = [ message for message in flow.websocket.messages if message.from_client ] elif flow_part == "websocket-response": if not flow.websocket: raise exceptions.CommandError( f"Flow part is websocket-response, but flow is not a websocket" ) messages = [ message for message in flow.websocket.messages if not message.from_client ] if not messages: raise exceptions.CommandError( f"Could not find any messages for flow part: {flow_part}" ) message_hash = _message_hash(messages[0].content, messages[0], flow) if typename == "(clear)": logging.info("popping message hash") self.typedef_lookup.pop(message_hash, None) self._save_project_file() self._refresh_view() return # Validate that we can decode all messages with our type for message in messages: try: _decode_protobuf( message.content, typename, self.bbpb_config, fallback=False ) except BlackboxProtobufException as ex: raise exceptions.CommandError( f"Error applying type name {typename} to part {flow_part}: {ex}" ) # Success self.typedef_lookup[message_hash] = typename self._save_project_file() self._refresh_view() @command.command("bbpb.new_type") @command.argument("flow_part", type=types.Choice("bbpb.options.edit_type_part")) @command.argument("typename", type=str) def bbpb_new_type(self, flow_part: str, typename: str) -> None: if typename == "(clear)": raise exceptions.CommandError(f"Error: Typename {typename} is not valid.") typedef, message_hash = self._resolve_type(flow_part) blackboxprotobuf.lib.api._strip_typedef_annotations(typedef) self.typedef_lookup[message_hash] = typename self.bbpb_config.known_types[typename] = typedef self._save_project_file() self._refresh_view() def _resolve_type(self, flow_part): flow = ctx.master.view.focus.flow if not flow: raise exceptions.CommandError("No flow selected.") if flow_part.startswith("request") or flow_part.startswith("response"): if flow_part == "request-body": message = flow.request elif flow_part == "response-body": if not flow.response: raise exceptions.CommandError( f"Flow part is response-body, but flow has no response" ) message = flow.response message_hash = _message_hash(message.content, message, flow) saved_typedef = self.typedef_lookup.get(message_hash) message_json, typedef, encoding_alg = _decode_protobuf( message.content, saved_typedef, self.bbpb_config ) elif flow_part.startswith("websocket"): # Websockets don't have a single typedef to edit # Instead, we are going to build a typedef based on all the messages if flow_part == "websocket-request": if not flow.websocket: raise exceptions.CommandError( f"Flow part is websocket-request, but flow is not a websocket" ) messages = [ message for message in flow.websocket.messages if message.from_client ] elif flow_part == "websocket-response": if not flow.websocket: raise exceptions.CommandError( f"Flow part is websocket-response, but flow is not a websocket" ) messages = [ message for message in flow.websocket.messages if not message.from_client ] if not messages: raise exceptions.CommandError( f"Could not find any messages for flow part: {flow_part}" ) message_hash = _message_hash(messages[0].content, messages[0], flow) saved_typedef = self.typedef_lookup.get(message_hash) try: typedef = saved_typedef message_jsons = [] for message in messages: message_json, typedef, encoding_alg = _decode_protobuf( message.content, typedef, self.bbpb_config, fallback=False ) message_jsons.append(message_json) except BlackboxProtobufException: typedef = {} message_jsons = [] for message in messages: message_json, typedef, encoding_alg = _decode_protobuf( message.content, typedef, self.bbpb_config, fallback=False ) message_jsons.append(message_json) else: raise exceptions.CommandError(f"Got unknown flow_part: {flow_part}") return typedef, message_hash @command.command("bbpb.del_type") @command.argument("typename", type=types.Choice("bbpb.options.known_types")) def bbpb_del_type(self, typename: str) -> None: if typename not in self.bbpb_config.known_types: raise exceptions.CommandError(f"Error: Type {typename} is not known") self.bbpb_config.known_types.pop(typename, None) keys_to_remove = [ key for key, value in self.typedef_lookup.items() if value == typename ] for key in keys_to_remove: self.typedef_lookup.pop(key, None) self._save_project_file() self._refresh_view() @command.command("bbpb.options.edit_part") def bbpb_options_edit_part(self) -> Sequence[str]: flow = ctx.master.view.focus.flow if not flow: raise exceptions.CommandError("No flow selected.") # Prompts the user for the section to edit if flow.websocket: if len(flow.websocket.messages) > 0: return ["websocket"] else: return [] elif flow.response: return [ "request-body", "response-body", ] else: return ["request-body"] @command.command("bbpb.options.edit_type_part") def bbpb_options_edit_type_part(self) -> Sequence[str]: flow = ctx.master.view.focus.flow if flow.websocket: return ["websocket-request", "websocket-response"] if flow.response: return [ "request-body", "response-body", ] return ["request-body"] @command.command("bbpb.options.known_types") def bbpb_options_known_types(self) -> Sequence[str]: typenames = list(self.bbpb_config.known_types.keys()) return typenames + ["(clear)"] def _refresh_view(self): ctx.master.window.stacks[0].windows["flowview"].body.contentview_changed(None) @command.command("bbpb.project.load") def bbpb_project_load(self, project_file: str) -> None: # TODO would be good to have errors propagated here self._load_project_file(project_file) @command.command("bbpb.project.save") def bbpb_project_save(self, project_file: str) -> None: # TODO would be good to have errors propagated here self._save_project_file(project_file) class BlackboxProtobufView(contentviews.View): name = "Blackbox Protobuf" def __init__(self, addon: BlackboxProtobufAddon): self.addon = addon def __call__( self, data: bytes, *, content_type: str | None = None, flow: flow.Flow | None = None, http_message: http.Message | None = None, **unknown_metadata, ) -> contentviews.TViewResult: # No support for TCP or UDP flows if not isinstance(flow, http.HTTPFlow): return None if len(data) == 0: return # message_hash is for looking up the appropiate typedef for this request, based on URL and type of message message_hash = _message_hash(data, http_message, flow) typedef = self.addon.typedef_lookup.get(message_hash) message, typedef_out, encoding_alg = _decode_protobuf( data, typedef, self.addon.bbpb_config ) title = "Protobuf" if isinstance(typedef, str): title += f" | Type: {typedef}" else: title += f" | Type: anonymous" return title, contentviews.format_text(message) def render_priority( self, data: bytes, *, content_type: str | None = None, flow: flow.Flow | None = None, http_message: http.Message | None = None, **unknown_metadata, ) -> float: if content_type: if "protobuf" in content_type or "grpc" in content_type: return 2 else: return 0 # We don't know if we can decode protobuf or not, so let's elect # ourselves for all websockets if flow.websocket: return 1 return 0 # This could be improved by taking is_request from the client for some cases def _message_hash( data: bytes, message: http.Message | websocket.WebSocketMessage | None, flow: flow.Flow | None, ): if isinstance(message, http.Request): return f"request|{flow.request.url}" elif isinstance(message, http.Response): return f"response|{flow.request.url}" elif flow.websocket: if message is None or not isinstance(message, websocket.WebSocketMessage): # TODO this is really hacky and might be wasted cycles if there are # a lot of messages # Mitmproxy won't give us the WebSocketMessage message type for # content views, so we can't tell which direction it's going from # just the message. try: message = next( (m for m in flow.websocket.messages if m.content == data) ) except StopIteration: logging.warn( "Message hashing couldn't find matching message in flow.websocket.messages" ) message = None # Default to request if we never figured out the message if message and not message.from_client: return f"websocket-response|{flow.request.url}" else: return f"websocket-request|{flow.request.url}" else: logging.warn( f"BBPB content view got a view that was not websocket, request or response: {type(http_message)}" ) return None def _decode_protobuf(data, typedef, config, fallback=True): try: decoders = payloads.find_decoders(data) for decoder in decoders: try: protobuf_data, encoding_alg = decoder(data) except BlackboxProtobufException: continue try: message, typedef_out = blackboxprotobuf.lib.protobuf_to_json( protobuf_data, typedef, config=config ) return message, typedef_out, encoding_alg except BlackboxProtobufException as exc: if encoding_alg == "none": raise exc continue except BlackboxProtobufException as exc: if typedef and fallback: return _decode_protobuf(data, {}, config) else: raise exc raise BlackboxProtobufException( 'Failed to decode protobuf, but did not catch "none" decoder. This should never be hit' ) # This function spawns an editor for a text file. Once the file is saved, it # calls `callback` with the text saved by the user. If `callback` throws an # exception, this function will present the error to the user and allow the # user to either keep editing the payload, reset the payload to the original # text and edit again, or give up and exit with no changes # # The UI requires that all the logic on the modified text is embedded in a # callback, I couldn't find any way to get a user choice without a callback # # This function is slightly cursed in that we have to use a custom chooser # class to allow recursive chooser calls, otherwise it only allows a single # prompt per command. Hopefully I can find a better way to handle this down the # road. T = TypeVar("T") def _spawn_validating_editor( text: str, callback: Callable[[str], None], original_text: Optional[str] = None ) -> None: user_text = ctx.master.spawn_editor(text) if original_text is None: original_text = text try: callback(user_text) signals.pop_view_state.send() except Exception as exc: options = ["Continue Editing", "Reset Payload And Edit", "Exit"] def choose_callback(action: str): if action == "Continue Editing": # Keep editing the text that failed _spawn_validating_editor(user_text, callback, original_text) elif action == "Reset Payload And Edit": # Edit the original text instead _spawn_validating_editor(original_text, callback) elif action == "Exit": # Just return signals.pop_view_state.send() return else: raise Exception( f"Got unknown option in validating editor menu: {action}" ) signals.pop_view_state.send() ctx.master.overlay( RecursiveChooser( ctx.master, f"Error validating payload: {exc}", options, "", choose_callback, ) ) # Below is the exact keypress implementation from `overlay.Chooser`, but # without `signals.pop_view_state.send()` after the callbacks This allows us to # spawn overlays (such as another chooser) from the callback function. # With `overlay.Chooser`, it calls `signals.pop_view_state.send()` *after* the # callback, which pops off the new chooser set by the callback # # This shifts responsibility for calling `signals.pop_view_state.send()` to the # callback, which should call it on exit or before popping up a new chooser class RecursiveChooser(overlay.Chooser): def keypress(self, size, key): key = self.master.keymap.handle_only("chooser", key) choice = self.walker.choice_by_shortcut(key) if choice: self.callback(choice) return if key == "m_select": self.callback(self.choices[self.walker.index]) return elif key in ["q", "esc"]: signals.pop_view_state.send() return binding = self.master.keymap.get("global", key) # This is extremely awkward. We need a better way to match nav keys only. if binding and binding.command.startswith("console.nav"): self.master.keymap.handle("global", key) elif key in keymap.navkeys: return super().keypress(size, key) addons = [BlackboxProtobufAddon()] blackboxprotobuf-1.4.2/mitmproxy/deps000077700000000000000000000000001476504373500220642../burp/depsustar00rootroot00000000000000