././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1754975852.913846 llm-0.27.1/0000755000175100001660000000000015046547155012004 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/LICENSE0000644000175100001660000002613515046547145013017 0ustar00runnerdocker Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/MANIFEST.in0000644000175100001660000000002715046547145013540 0ustar00runnerdockerglobal-exclude tests/* ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1754975852.913846 llm-0.27.1/PKG-INFO0000644000175100001660000007325215046547155013112 0ustar00runnerdockerMetadata-Version: 2.4 Name: llm Version: 0.27.1 Summary: CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine. Author: Simon Willison License-Expression: Apache-2.0 Project-URL: Homepage, https://github.com/simonw/llm Project-URL: Documentation, https://llm.datasette.io/ Project-URL: Issues, https://github.com/simonw/llm/issues Project-URL: CI, https://github.com/simonw/llm/actions Project-URL: Changelog, https://github.com/simonw/llm/releases Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: Intended Audience :: End Users/Desktop Classifier: Intended Audience :: Science/Research Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence Classifier: Topic :: Text Processing :: Linguistic Classifier: Topic :: Utilities Requires-Python: >=3.9 Description-Content-Type: text/markdown License-File: LICENSE Requires-Dist: click Requires-Dist: condense-json>=0.1.3 Requires-Dist: openai>=1.55.3 Requires-Dist: click-default-group>=1.2.3 Requires-Dist: sqlite-utils>=3.37 Requires-Dist: sqlite-migrate>=0.1a2 Requires-Dist: pydantic>=2.0.0 Requires-Dist: PyYAML Requires-Dist: pluggy Requires-Dist: python-ulid Requires-Dist: setuptools Requires-Dist: pip Requires-Dist: pyreadline3; sys_platform == "win32" Requires-Dist: puremagic Provides-Extra: test Requires-Dist: build; extra == "test" Requires-Dist: click<8.2.0; extra == "test" Requires-Dist: pytest; extra == "test" Requires-Dist: numpy; extra == "test" Requires-Dist: pytest-httpx>=0.33.0; extra == "test" Requires-Dist: pytest-asyncio; extra == "test" Requires-Dist: cogapp; extra == "test" Requires-Dist: mypy>=1.10.0; extra == "test" Requires-Dist: black>=25.1.0; extra == "test" Requires-Dist: pytest-recording; extra == "test" Requires-Dist: ruff; extra == "test" Requires-Dist: syrupy; extra == "test" Requires-Dist: types-click; extra == "test" Requires-Dist: types-PyYAML; extra == "test" Requires-Dist: types-setuptools; extra == "test" Requires-Dist: llm-echo==0.3a3; extra == "test" Dynamic: license-file # LLM [![GitHub repo](https://img.shields.io/badge/github-repo-green)](https://github.com/simonw/llm) [![PyPI](https://img.shields.io/pypi/v/llm.svg)](https://pypi.org/project/llm/) [![Changelog](https://img.shields.io/github/v/release/simonw/llm?include_prereleases&label=changelog)](https://llm.datasette.io/en/stable/changelog.html) [![Tests](https://github.com/simonw/llm/workflows/Test/badge.svg)](https://github.com/simonw/llm/actions?query=workflow%3ATest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm/blob/main/LICENSE) [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Meta’s Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). With LLM you can: - [Run prompts from the command-line](https://llm.datasette.io/en/stable/usage.html#usage-executing-prompts) - [Store prompts and responses in SQLite](https://llm.datasette.io/en/stable/logging.html#logging) - [Generate and store embeddings](https://llm.datasette.io/en/stable/embeddings/index.html#embeddings) - [Extract structured content from text and images](https://llm.datasette.io/en/stable/schemas.html#schemas) - [Grant models the ability to execute tools](https://llm.datasette.io/en/stable/tools.html#tools) - … and much, much more ## Quick start First, install LLM using `pip` or Homebrew or `pipx` or `uv`: ```bash pip install llm ``` Or with Homebrew (see [warning note](https://llm.datasette.io/en/stable/setup.html#homebrew-warning)): ```bash brew install llm ``` Or with [pipx](https://pypa.github.io/pipx/): ```bash pipx install llm ``` Or with [uv](https://docs.astral.sh/uv/guides/tools/) ```bash uv tool install llm ``` If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash # Paste your OpenAI API key into this llm keys set openai # Run a prompt (with the default gpt-4o-mini model) llm "Ten fun names for a pet pelican" # Extract text from an image llm "extract text" -a scanned-document.jpg # Use a system prompt against a file cat myfile.py | llm -s "Explain this code" ``` Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic](https://console.anthropic.com/) with their respective plugins: ```bash llm install llm-gemini llm keys set gemini # Paste Gemini API key here llm -m gemini-2.0-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here llm -m claude-4-opus 'Impress me with wild facts about turnips' ``` You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/): ```bash # Install the plugin llm install llm-ollama # Download and run a prompt against the Orca Mini 7B model ollama pull llama3.2:latest llm -m llama3.2:latest 'What is the capital of France?' ``` To start [an interactive chat](https://llm.datasette.io/en/stable/usage.html#usage-chat) with a model, use `llm chat`: ```bash llm chat -m gpt-4.1 ``` ```default Chatting with gpt-4.1 Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt. Type '!fragment [ ...]' to insert one or more fragments > Tell me a joke about a pelican Why don't pelicans like to tip waiters? Because they always have a big bill! ``` More background on this project: - [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) - [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) - [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) - [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) - [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) - [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) - [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents * [Setup](https://llm.datasette.io/en/stable/setup.html) * [Installation](https://llm.datasette.io/en/stable/setup.html#installation) * [Upgrading to the latest version](https://llm.datasette.io/en/stable/setup.html#upgrading-to-the-latest-version) * [Using uvx](https://llm.datasette.io/en/stable/setup.html#using-uvx) * [A note about Homebrew and PyTorch](https://llm.datasette.io/en/stable/setup.html#a-note-about-homebrew-and-pytorch) * [Installing plugins](https://llm.datasette.io/en/stable/setup.html#installing-plugins) * [API key management](https://llm.datasette.io/en/stable/setup.html#api-key-management) * [Saving and using stored keys](https://llm.datasette.io/en/stable/setup.html#saving-and-using-stored-keys) * [Passing keys using the –key option](https://llm.datasette.io/en/stable/setup.html#passing-keys-using-the-key-option) * [Keys in environment variables](https://llm.datasette.io/en/stable/setup.html#keys-in-environment-variables) * [Configuration](https://llm.datasette.io/en/stable/setup.html#configuration) * [Setting a custom default model](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-default-model) * [Setting a custom directory location](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-directory-location) * [Turning SQLite logging on and off](https://llm.datasette.io/en/stable/setup.html#turning-sqlite-logging-on-and-off) * [Usage](https://llm.datasette.io/en/stable/usage.html) * [Executing a prompt](https://llm.datasette.io/en/stable/usage.html#executing-a-prompt) * [Model options](https://llm.datasette.io/en/stable/usage.html#model-options) * [Attachments](https://llm.datasette.io/en/stable/usage.html#attachments) * [System prompts](https://llm.datasette.io/en/stable/usage.html#system-prompts) * [Tools](https://llm.datasette.io/en/stable/usage.html#tools) * [Extracting fenced code blocks](https://llm.datasette.io/en/stable/usage.html#extracting-fenced-code-blocks) * [Schemas](https://llm.datasette.io/en/stable/usage.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/usage.html#fragments) * [Continuing a conversation](https://llm.datasette.io/en/stable/usage.html#continuing-a-conversation) * [Tips for using LLM with Bash or Zsh](https://llm.datasette.io/en/stable/usage.html#tips-for-using-llm-with-bash-or-zsh) * [Completion prompts](https://llm.datasette.io/en/stable/usage.html#completion-prompts) * [Starting an interactive chat](https://llm.datasette.io/en/stable/usage.html#starting-an-interactive-chat) * [Listing available models](https://llm.datasette.io/en/stable/usage.html#listing-available-models) * [Setting default options for models](https://llm.datasette.io/en/stable/usage.html#setting-default-options-for-models) * [OpenAI models](https://llm.datasette.io/en/stable/openai-models.html) * [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration) * [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models) * [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features) * [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models) * [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models) * [Adding more OpenAI models](https://llm.datasette.io/en/stable/openai-models.html#adding-more-openai-models) * [Other models](https://llm.datasette.io/en/stable/other-models.html) * [Installing and using a local model](https://llm.datasette.io/en/stable/other-models.html#installing-and-using-a-local-model) * [OpenAI-compatible models](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models) * [Extra HTTP headers](https://llm.datasette.io/en/stable/other-models.html#extra-http-headers) * [Tools](https://llm.datasette.io/en/stable/tools.html) * [How tools work](https://llm.datasette.io/en/stable/tools.html#how-tools-work) * [Trying out tools](https://llm.datasette.io/en/stable/tools.html#trying-out-tools) * [LLM’s implementation of tools](https://llm.datasette.io/en/stable/tools.html#llm-s-implementation-of-tools) * [Default tools](https://llm.datasette.io/en/stable/tools.html#default-tools) * [Tips for implementing tools](https://llm.datasette.io/en/stable/tools.html#tips-for-implementing-tools) * [Schemas](https://llm.datasette.io/en/stable/schemas.html) * [Schemas tutorial](https://llm.datasette.io/en/stable/schemas.html#schemas-tutorial) * [Getting started with dogs](https://llm.datasette.io/en/stable/schemas.html#getting-started-with-dogs) * [Extracting people from a news articles](https://llm.datasette.io/en/stable/schemas.html#extracting-people-from-a-news-articles) * [Using JSON schemas](https://llm.datasette.io/en/stable/schemas.html#using-json-schemas) * [Ways to specify a schema](https://llm.datasette.io/en/stable/schemas.html#ways-to-specify-a-schema) * [Concise LLM schema syntax](https://llm.datasette.io/en/stable/schemas.html#concise-llm-schema-syntax) * [Saving reusable schemas in templates](https://llm.datasette.io/en/stable/schemas.html#saving-reusable-schemas-in-templates) * [Browsing logged JSON objects created using schemas](https://llm.datasette.io/en/stable/schemas.html#browsing-logged-json-objects-created-using-schemas) * [Templates](https://llm.datasette.io/en/stable/templates.html) * [Getting started with –save](https://llm.datasette.io/en/stable/templates.html#getting-started-with-save) * [Using a template](https://llm.datasette.io/en/stable/templates.html#using-a-template) * [Listing available templates](https://llm.datasette.io/en/stable/templates.html#listing-available-templates) * [Templates as YAML files](https://llm.datasette.io/en/stable/templates.html#templates-as-yaml-files) * [System prompts](https://llm.datasette.io/en/stable/templates.html#system-prompts) * [Fragments](https://llm.datasette.io/en/stable/templates.html#fragments) * [Options](https://llm.datasette.io/en/stable/templates.html#options) * [Tools](https://llm.datasette.io/en/stable/templates.html#tools) * [Schemas](https://llm.datasette.io/en/stable/templates.html#schemas) * [Additional template variables](https://llm.datasette.io/en/stable/templates.html#additional-template-variables) * [Specifying default parameters](https://llm.datasette.io/en/stable/templates.html#specifying-default-parameters) * [Configuring code extraction](https://llm.datasette.io/en/stable/templates.html#configuring-code-extraction) * [Setting a default model for a template](https://llm.datasette.io/en/stable/templates.html#setting-a-default-model-for-a-template) * [Template loaders from plugins](https://llm.datasette.io/en/stable/templates.html#template-loaders-from-plugins) * [Fragments](https://llm.datasette.io/en/stable/fragments.html) * [Using fragments in a prompt](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-a-prompt) * [Using fragments in chat](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-chat) * [Browsing fragments](https://llm.datasette.io/en/stable/fragments.html#browsing-fragments) * [Setting aliases for fragments](https://llm.datasette.io/en/stable/fragments.html#setting-aliases-for-fragments) * [Viewing fragments in your logs](https://llm.datasette.io/en/stable/fragments.html#viewing-fragments-in-your-logs) * [Using fragments from plugins](https://llm.datasette.io/en/stable/fragments.html#using-fragments-from-plugins) * [Listing available fragment prefixes](https://llm.datasette.io/en/stable/fragments.html#listing-available-fragment-prefixes) * [Model aliases](https://llm.datasette.io/en/stable/aliases.html) * [Listing aliases](https://llm.datasette.io/en/stable/aliases.html#listing-aliases) * [Adding a new alias](https://llm.datasette.io/en/stable/aliases.html#adding-a-new-alias) * [Removing an alias](https://llm.datasette.io/en/stable/aliases.html#removing-an-alias) * [Viewing the aliases file](https://llm.datasette.io/en/stable/aliases.html#viewing-the-aliases-file) * [Embeddings](https://llm.datasette.io/en/stable/embeddings/index.html) * [Embedding with the CLI](https://llm.datasette.io/en/stable/embeddings/cli.html) * [llm embed](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed) * [llm embed-multi](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-multi) * [llm similar](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-similar) * [llm embed-models](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-models) * [llm collections list](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-list) * [llm collections delete](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-delete) * [Using embeddings from Python](https://llm.datasette.io/en/stable/embeddings/python-api.html) * [Working with collections](https://llm.datasette.io/en/stable/embeddings/python-api.html#working-with-collections) * [Retrieving similar items](https://llm.datasette.io/en/stable/embeddings/python-api.html#retrieving-similar-items) * [SQL schema](https://llm.datasette.io/en/stable/embeddings/python-api.html#sql-schema) * [Writing plugins to add new embedding models](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html) * [Embedding binary content](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#embedding-binary-content) * [Embedding storage format](https://llm.datasette.io/en/stable/embeddings/storage.html) * [Plugins](https://llm.datasette.io/en/stable/plugins/index.html) * [Installing plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html) * [Listing installed plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#listing-installed-plugins) * [Running with a subset of plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#running-with-a-subset-of-plugins) * [Plugin directory](https://llm.datasette.io/en/stable/plugins/directory.html) * [Local models](https://llm.datasette.io/en/stable/plugins/directory.html#local-models) * [Remote APIs](https://llm.datasette.io/en/stable/plugins/directory.html#remote-apis) * [Tools](https://llm.datasette.io/en/stable/plugins/directory.html#tools) * [Fragments and template loaders](https://llm.datasette.io/en/stable/plugins/directory.html#fragments-and-template-loaders) * [Embedding models](https://llm.datasette.io/en/stable/plugins/directory.html#embedding-models) * [Extra commands](https://llm.datasette.io/en/stable/plugins/directory.html#extra-commands) * [Just for fun](https://llm.datasette.io/en/stable/plugins/directory.html#just-for-fun) * [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html) * [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli) * [register_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register) * [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register) * [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register) * [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register) * [register_fragment_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-fragment-loaders-register) * [Developing a model plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html) * [The initial structure of the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#the-initial-structure-of-the-plugin) * [Installing your plugin to try it out](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#installing-your-plugin-to-try-it-out) * [Building the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#building-the-markov-chain) * [Executing the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#executing-the-markov-chain) * [Adding that to the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-that-to-the-plugin) * [Understanding execute()](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#understanding-execute) * [Prompts and responses are logged to the database](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#prompts-and-responses-are-logged-to-the-database) * [Adding options](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-options) * [Distributing your plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#distributing-your-plugin) * [GitHub repositories](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#github-repositories) * [Publishing plugins to PyPI](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#publishing-plugins-to-pypi) * [Adding metadata](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-metadata) * [What to do if it breaks](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#what-to-do-if-it-breaks) * [Advanced model plugins](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html) * [Tip: lazily load expensive dependencies](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tip-lazily-load-expensive-dependencies) * [Models that accept API keys](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#models-that-accept-api-keys) * [Async models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#async-models) * [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas) * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) * [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names) * [LLM_RAISE_ERRORS](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#llm-raise-errors) * [Utility functions for plugins](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html) * [llm.get_key()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-get-key) * [llm.user_dir()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-user-dir) * [llm.ModelError](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-modelerror) * [Response.fake()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#response-fake) * [Python API](https://llm.datasette.io/en/stable/python-api.html) * [Basic prompt execution](https://llm.datasette.io/en/stable/python-api.html#basic-prompt-execution) * [System prompts](https://llm.datasette.io/en/stable/python-api.html#system-prompts) * [Attachments](https://llm.datasette.io/en/stable/python-api.html#attachments) * [Tools](https://llm.datasette.io/en/stable/python-api.html#tools) * [Schemas](https://llm.datasette.io/en/stable/python-api.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/python-api.html#fragments) * [Model options](https://llm.datasette.io/en/stable/python-api.html#model-options) * [Passing an API key](https://llm.datasette.io/en/stable/python-api.html#passing-an-api-key) * [Models from plugins](https://llm.datasette.io/en/stable/python-api.html#models-from-plugins) * [Accessing the underlying JSON](https://llm.datasette.io/en/stable/python-api.html#accessing-the-underlying-json) * [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage) * [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses) * [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models) * [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async) * [Tool use for async models](https://llm.datasette.io/en/stable/python-api.html#tool-use-for-async-models) * [Conversations](https://llm.datasette.io/en/stable/python-api.html#conversations) * [Conversations using tools](https://llm.datasette.io/en/stable/python-api.html#conversations-using-tools) * [Listing models](https://llm.datasette.io/en/stable/python-api.html#listing-models) * [Running code when a response has completed](https://llm.datasette.io/en/stable/python-api.html#running-code-when-a-response-has-completed) * [Other functions](https://llm.datasette.io/en/stable/python-api.html#other-functions) * [set_alias(alias, model_id)](https://llm.datasette.io/en/stable/python-api.html#set-alias-alias-model-id) * [remove_alias(alias)](https://llm.datasette.io/en/stable/python-api.html#remove-alias-alias) * [set_default_model(alias)](https://llm.datasette.io/en/stable/python-api.html#set-default-model-alias) * [get_default_model()](https://llm.datasette.io/en/stable/python-api.html#get-default-model) * [set_default_embedding_model(alias) and get_default_embedding_model()](https://llm.datasette.io/en/stable/python-api.html#set-default-embedding-model-alias-and-get-default-embedding-model) * [Logging to SQLite](https://llm.datasette.io/en/stable/logging.html) * [Viewing the logs](https://llm.datasette.io/en/stable/logging.html#viewing-the-logs) * [-s/–short mode](https://llm.datasette.io/en/stable/logging.html#s-short-mode) * [Logs for a conversation](https://llm.datasette.io/en/stable/logging.html#logs-for-a-conversation) * [Searching the logs](https://llm.datasette.io/en/stable/logging.html#searching-the-logs) * [Filtering past a specific ID](https://llm.datasette.io/en/stable/logging.html#filtering-past-a-specific-id) * [Filtering by model](https://llm.datasette.io/en/stable/logging.html#filtering-by-model) * [Filtering by prompts that used specific fragments](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-fragments) * [Filtering by prompts that used specific tools](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-tools) * [Browsing data collected using schemas](https://llm.datasette.io/en/stable/logging.html#browsing-data-collected-using-schemas) * [Browsing logs using Datasette](https://llm.datasette.io/en/stable/logging.html#browsing-logs-using-datasette) * [Backing up your database](https://llm.datasette.io/en/stable/logging.html#backing-up-your-database) * [SQL schema](https://llm.datasette.io/en/stable/logging.html#sql-schema) * [Related tools](https://llm.datasette.io/en/stable/related-tools.html) * [strip-tags](https://llm.datasette.io/en/stable/related-tools.html#strip-tags) * [ttok](https://llm.datasette.io/en/stable/related-tools.html#ttok) * [Symbex](https://llm.datasette.io/en/stable/related-tools.html#symbex) * [CLI reference](https://llm.datasette.io/en/stable/help.html) * [llm –help](https://llm.datasette.io/en/stable/help.html#llm-help) * [llm prompt –help](https://llm.datasette.io/en/stable/help.html#llm-prompt-help) * [llm chat –help](https://llm.datasette.io/en/stable/help.html#llm-chat-help) * [llm keys –help](https://llm.datasette.io/en/stable/help.html#llm-keys-help) * [llm logs –help](https://llm.datasette.io/en/stable/help.html#llm-logs-help) * [llm models –help](https://llm.datasette.io/en/stable/help.html#llm-models-help) * [llm templates –help](https://llm.datasette.io/en/stable/help.html#llm-templates-help) * [llm schemas –help](https://llm.datasette.io/en/stable/help.html#llm-schemas-help) * [llm tools –help](https://llm.datasette.io/en/stable/help.html#llm-tools-help) * [llm aliases –help](https://llm.datasette.io/en/stable/help.html#llm-aliases-help) * [llm fragments –help](https://llm.datasette.io/en/stable/help.html#llm-fragments-help) * [llm plugins –help](https://llm.datasette.io/en/stable/help.html#llm-plugins-help) * [llm install –help](https://llm.datasette.io/en/stable/help.html#llm-install-help) * [llm uninstall –help](https://llm.datasette.io/en/stable/help.html#llm-uninstall-help) * [llm embed –help](https://llm.datasette.io/en/stable/help.html#llm-embed-help) * [llm embed-multi –help](https://llm.datasette.io/en/stable/help.html#llm-embed-multi-help) * [llm similar –help](https://llm.datasette.io/en/stable/help.html#llm-similar-help) * [llm embed-models –help](https://llm.datasette.io/en/stable/help.html#llm-embed-models-help) * [llm collections –help](https://llm.datasette.io/en/stable/help.html#llm-collections-help) * [llm openai –help](https://llm.datasette.io/en/stable/help.html#llm-openai-help) * [Contributing](https://llm.datasette.io/en/stable/contributing.html) * [Updating recorded HTTP API interactions and associated snapshots](https://llm.datasette.io/en/stable/contributing.html#updating-recorded-http-api-interactions-and-associated-snapshots) * [Debugging tricks](https://llm.datasette.io/en/stable/contributing.html#debugging-tricks) * [Documentation](https://llm.datasette.io/en/stable/contributing.html#documentation) * [Release process](https://llm.datasette.io/en/stable/contributing.html#release-process) * [Changelog](https://llm.datasette.io/en/stable/changelog.html) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/README.md0000644000175100001660000006641015046547145013271 0ustar00runnerdocker # LLM [![GitHub repo](https://img.shields.io/badge/github-repo-green)](https://github.com/simonw/llm) [![PyPI](https://img.shields.io/pypi/v/llm.svg)](https://pypi.org/project/llm/) [![Changelog](https://img.shields.io/github/v/release/simonw/llm?include_prereleases&label=changelog)](https://llm.datasette.io/en/stable/changelog.html) [![Tests](https://github.com/simonw/llm/workflows/Test/badge.svg)](https://github.com/simonw/llm/actions?query=workflow%3ATest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm/blob/main/LICENSE) [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Meta’s Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). With LLM you can: - [Run prompts from the command-line](https://llm.datasette.io/en/stable/usage.html#usage-executing-prompts) - [Store prompts and responses in SQLite](https://llm.datasette.io/en/stable/logging.html#logging) - [Generate and store embeddings](https://llm.datasette.io/en/stable/embeddings/index.html#embeddings) - [Extract structured content from text and images](https://llm.datasette.io/en/stable/schemas.html#schemas) - [Grant models the ability to execute tools](https://llm.datasette.io/en/stable/tools.html#tools) - … and much, much more ## Quick start First, install LLM using `pip` or Homebrew or `pipx` or `uv`: ```bash pip install llm ``` Or with Homebrew (see [warning note](https://llm.datasette.io/en/stable/setup.html#homebrew-warning)): ```bash brew install llm ``` Or with [pipx](https://pypa.github.io/pipx/): ```bash pipx install llm ``` Or with [uv](https://docs.astral.sh/uv/guides/tools/) ```bash uv tool install llm ``` If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash # Paste your OpenAI API key into this llm keys set openai # Run a prompt (with the default gpt-4o-mini model) llm "Ten fun names for a pet pelican" # Extract text from an image llm "extract text" -a scanned-document.jpg # Use a system prompt against a file cat myfile.py | llm -s "Explain this code" ``` Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic](https://console.anthropic.com/) with their respective plugins: ```bash llm install llm-gemini llm keys set gemini # Paste Gemini API key here llm -m gemini-2.0-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here llm -m claude-4-opus 'Impress me with wild facts about turnips' ``` You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/): ```bash # Install the plugin llm install llm-ollama # Download and run a prompt against the Orca Mini 7B model ollama pull llama3.2:latest llm -m llama3.2:latest 'What is the capital of France?' ``` To start [an interactive chat](https://llm.datasette.io/en/stable/usage.html#usage-chat) with a model, use `llm chat`: ```bash llm chat -m gpt-4.1 ``` ```default Chatting with gpt-4.1 Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt. Type '!fragment [ ...]' to insert one or more fragments > Tell me a joke about a pelican Why don't pelicans like to tip waiters? Because they always have a big bill! ``` More background on this project: - [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) - [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) - [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) - [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) - [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) - [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) - [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents * [Setup](https://llm.datasette.io/en/stable/setup.html) * [Installation](https://llm.datasette.io/en/stable/setup.html#installation) * [Upgrading to the latest version](https://llm.datasette.io/en/stable/setup.html#upgrading-to-the-latest-version) * [Using uvx](https://llm.datasette.io/en/stable/setup.html#using-uvx) * [A note about Homebrew and PyTorch](https://llm.datasette.io/en/stable/setup.html#a-note-about-homebrew-and-pytorch) * [Installing plugins](https://llm.datasette.io/en/stable/setup.html#installing-plugins) * [API key management](https://llm.datasette.io/en/stable/setup.html#api-key-management) * [Saving and using stored keys](https://llm.datasette.io/en/stable/setup.html#saving-and-using-stored-keys) * [Passing keys using the –key option](https://llm.datasette.io/en/stable/setup.html#passing-keys-using-the-key-option) * [Keys in environment variables](https://llm.datasette.io/en/stable/setup.html#keys-in-environment-variables) * [Configuration](https://llm.datasette.io/en/stable/setup.html#configuration) * [Setting a custom default model](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-default-model) * [Setting a custom directory location](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-directory-location) * [Turning SQLite logging on and off](https://llm.datasette.io/en/stable/setup.html#turning-sqlite-logging-on-and-off) * [Usage](https://llm.datasette.io/en/stable/usage.html) * [Executing a prompt](https://llm.datasette.io/en/stable/usage.html#executing-a-prompt) * [Model options](https://llm.datasette.io/en/stable/usage.html#model-options) * [Attachments](https://llm.datasette.io/en/stable/usage.html#attachments) * [System prompts](https://llm.datasette.io/en/stable/usage.html#system-prompts) * [Tools](https://llm.datasette.io/en/stable/usage.html#tools) * [Extracting fenced code blocks](https://llm.datasette.io/en/stable/usage.html#extracting-fenced-code-blocks) * [Schemas](https://llm.datasette.io/en/stable/usage.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/usage.html#fragments) * [Continuing a conversation](https://llm.datasette.io/en/stable/usage.html#continuing-a-conversation) * [Tips for using LLM with Bash or Zsh](https://llm.datasette.io/en/stable/usage.html#tips-for-using-llm-with-bash-or-zsh) * [Completion prompts](https://llm.datasette.io/en/stable/usage.html#completion-prompts) * [Starting an interactive chat](https://llm.datasette.io/en/stable/usage.html#starting-an-interactive-chat) * [Listing available models](https://llm.datasette.io/en/stable/usage.html#listing-available-models) * [Setting default options for models](https://llm.datasette.io/en/stable/usage.html#setting-default-options-for-models) * [OpenAI models](https://llm.datasette.io/en/stable/openai-models.html) * [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration) * [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models) * [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features) * [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models) * [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models) * [Adding more OpenAI models](https://llm.datasette.io/en/stable/openai-models.html#adding-more-openai-models) * [Other models](https://llm.datasette.io/en/stable/other-models.html) * [Installing and using a local model](https://llm.datasette.io/en/stable/other-models.html#installing-and-using-a-local-model) * [OpenAI-compatible models](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models) * [Extra HTTP headers](https://llm.datasette.io/en/stable/other-models.html#extra-http-headers) * [Tools](https://llm.datasette.io/en/stable/tools.html) * [How tools work](https://llm.datasette.io/en/stable/tools.html#how-tools-work) * [Trying out tools](https://llm.datasette.io/en/stable/tools.html#trying-out-tools) * [LLM’s implementation of tools](https://llm.datasette.io/en/stable/tools.html#llm-s-implementation-of-tools) * [Default tools](https://llm.datasette.io/en/stable/tools.html#default-tools) * [Tips for implementing tools](https://llm.datasette.io/en/stable/tools.html#tips-for-implementing-tools) * [Schemas](https://llm.datasette.io/en/stable/schemas.html) * [Schemas tutorial](https://llm.datasette.io/en/stable/schemas.html#schemas-tutorial) * [Getting started with dogs](https://llm.datasette.io/en/stable/schemas.html#getting-started-with-dogs) * [Extracting people from a news articles](https://llm.datasette.io/en/stable/schemas.html#extracting-people-from-a-news-articles) * [Using JSON schemas](https://llm.datasette.io/en/stable/schemas.html#using-json-schemas) * [Ways to specify a schema](https://llm.datasette.io/en/stable/schemas.html#ways-to-specify-a-schema) * [Concise LLM schema syntax](https://llm.datasette.io/en/stable/schemas.html#concise-llm-schema-syntax) * [Saving reusable schemas in templates](https://llm.datasette.io/en/stable/schemas.html#saving-reusable-schemas-in-templates) * [Browsing logged JSON objects created using schemas](https://llm.datasette.io/en/stable/schemas.html#browsing-logged-json-objects-created-using-schemas) * [Templates](https://llm.datasette.io/en/stable/templates.html) * [Getting started with –save](https://llm.datasette.io/en/stable/templates.html#getting-started-with-save) * [Using a template](https://llm.datasette.io/en/stable/templates.html#using-a-template) * [Listing available templates](https://llm.datasette.io/en/stable/templates.html#listing-available-templates) * [Templates as YAML files](https://llm.datasette.io/en/stable/templates.html#templates-as-yaml-files) * [System prompts](https://llm.datasette.io/en/stable/templates.html#system-prompts) * [Fragments](https://llm.datasette.io/en/stable/templates.html#fragments) * [Options](https://llm.datasette.io/en/stable/templates.html#options) * [Tools](https://llm.datasette.io/en/stable/templates.html#tools) * [Schemas](https://llm.datasette.io/en/stable/templates.html#schemas) * [Additional template variables](https://llm.datasette.io/en/stable/templates.html#additional-template-variables) * [Specifying default parameters](https://llm.datasette.io/en/stable/templates.html#specifying-default-parameters) * [Configuring code extraction](https://llm.datasette.io/en/stable/templates.html#configuring-code-extraction) * [Setting a default model for a template](https://llm.datasette.io/en/stable/templates.html#setting-a-default-model-for-a-template) * [Template loaders from plugins](https://llm.datasette.io/en/stable/templates.html#template-loaders-from-plugins) * [Fragments](https://llm.datasette.io/en/stable/fragments.html) * [Using fragments in a prompt](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-a-prompt) * [Using fragments in chat](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-chat) * [Browsing fragments](https://llm.datasette.io/en/stable/fragments.html#browsing-fragments) * [Setting aliases for fragments](https://llm.datasette.io/en/stable/fragments.html#setting-aliases-for-fragments) * [Viewing fragments in your logs](https://llm.datasette.io/en/stable/fragments.html#viewing-fragments-in-your-logs) * [Using fragments from plugins](https://llm.datasette.io/en/stable/fragments.html#using-fragments-from-plugins) * [Listing available fragment prefixes](https://llm.datasette.io/en/stable/fragments.html#listing-available-fragment-prefixes) * [Model aliases](https://llm.datasette.io/en/stable/aliases.html) * [Listing aliases](https://llm.datasette.io/en/stable/aliases.html#listing-aliases) * [Adding a new alias](https://llm.datasette.io/en/stable/aliases.html#adding-a-new-alias) * [Removing an alias](https://llm.datasette.io/en/stable/aliases.html#removing-an-alias) * [Viewing the aliases file](https://llm.datasette.io/en/stable/aliases.html#viewing-the-aliases-file) * [Embeddings](https://llm.datasette.io/en/stable/embeddings/index.html) * [Embedding with the CLI](https://llm.datasette.io/en/stable/embeddings/cli.html) * [llm embed](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed) * [llm embed-multi](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-multi) * [llm similar](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-similar) * [llm embed-models](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-models) * [llm collections list](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-list) * [llm collections delete](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-delete) * [Using embeddings from Python](https://llm.datasette.io/en/stable/embeddings/python-api.html) * [Working with collections](https://llm.datasette.io/en/stable/embeddings/python-api.html#working-with-collections) * [Retrieving similar items](https://llm.datasette.io/en/stable/embeddings/python-api.html#retrieving-similar-items) * [SQL schema](https://llm.datasette.io/en/stable/embeddings/python-api.html#sql-schema) * [Writing plugins to add new embedding models](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html) * [Embedding binary content](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#embedding-binary-content) * [Embedding storage format](https://llm.datasette.io/en/stable/embeddings/storage.html) * [Plugins](https://llm.datasette.io/en/stable/plugins/index.html) * [Installing plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html) * [Listing installed plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#listing-installed-plugins) * [Running with a subset of plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#running-with-a-subset-of-plugins) * [Plugin directory](https://llm.datasette.io/en/stable/plugins/directory.html) * [Local models](https://llm.datasette.io/en/stable/plugins/directory.html#local-models) * [Remote APIs](https://llm.datasette.io/en/stable/plugins/directory.html#remote-apis) * [Tools](https://llm.datasette.io/en/stable/plugins/directory.html#tools) * [Fragments and template loaders](https://llm.datasette.io/en/stable/plugins/directory.html#fragments-and-template-loaders) * [Embedding models](https://llm.datasette.io/en/stable/plugins/directory.html#embedding-models) * [Extra commands](https://llm.datasette.io/en/stable/plugins/directory.html#extra-commands) * [Just for fun](https://llm.datasette.io/en/stable/plugins/directory.html#just-for-fun) * [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html) * [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli) * [register_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register) * [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register) * [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register) * [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register) * [register_fragment_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-fragment-loaders-register) * [Developing a model plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html) * [The initial structure of the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#the-initial-structure-of-the-plugin) * [Installing your plugin to try it out](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#installing-your-plugin-to-try-it-out) * [Building the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#building-the-markov-chain) * [Executing the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#executing-the-markov-chain) * [Adding that to the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-that-to-the-plugin) * [Understanding execute()](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#understanding-execute) * [Prompts and responses are logged to the database](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#prompts-and-responses-are-logged-to-the-database) * [Adding options](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-options) * [Distributing your plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#distributing-your-plugin) * [GitHub repositories](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#github-repositories) * [Publishing plugins to PyPI](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#publishing-plugins-to-pypi) * [Adding metadata](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-metadata) * [What to do if it breaks](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#what-to-do-if-it-breaks) * [Advanced model plugins](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html) * [Tip: lazily load expensive dependencies](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tip-lazily-load-expensive-dependencies) * [Models that accept API keys](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#models-that-accept-api-keys) * [Async models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#async-models) * [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas) * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) * [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names) * [LLM_RAISE_ERRORS](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#llm-raise-errors) * [Utility functions for plugins](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html) * [llm.get_key()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-get-key) * [llm.user_dir()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-user-dir) * [llm.ModelError](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-modelerror) * [Response.fake()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#response-fake) * [Python API](https://llm.datasette.io/en/stable/python-api.html) * [Basic prompt execution](https://llm.datasette.io/en/stable/python-api.html#basic-prompt-execution) * [System prompts](https://llm.datasette.io/en/stable/python-api.html#system-prompts) * [Attachments](https://llm.datasette.io/en/stable/python-api.html#attachments) * [Tools](https://llm.datasette.io/en/stable/python-api.html#tools) * [Schemas](https://llm.datasette.io/en/stable/python-api.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/python-api.html#fragments) * [Model options](https://llm.datasette.io/en/stable/python-api.html#model-options) * [Passing an API key](https://llm.datasette.io/en/stable/python-api.html#passing-an-api-key) * [Models from plugins](https://llm.datasette.io/en/stable/python-api.html#models-from-plugins) * [Accessing the underlying JSON](https://llm.datasette.io/en/stable/python-api.html#accessing-the-underlying-json) * [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage) * [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses) * [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models) * [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async) * [Tool use for async models](https://llm.datasette.io/en/stable/python-api.html#tool-use-for-async-models) * [Conversations](https://llm.datasette.io/en/stable/python-api.html#conversations) * [Conversations using tools](https://llm.datasette.io/en/stable/python-api.html#conversations-using-tools) * [Listing models](https://llm.datasette.io/en/stable/python-api.html#listing-models) * [Running code when a response has completed](https://llm.datasette.io/en/stable/python-api.html#running-code-when-a-response-has-completed) * [Other functions](https://llm.datasette.io/en/stable/python-api.html#other-functions) * [set_alias(alias, model_id)](https://llm.datasette.io/en/stable/python-api.html#set-alias-alias-model-id) * [remove_alias(alias)](https://llm.datasette.io/en/stable/python-api.html#remove-alias-alias) * [set_default_model(alias)](https://llm.datasette.io/en/stable/python-api.html#set-default-model-alias) * [get_default_model()](https://llm.datasette.io/en/stable/python-api.html#get-default-model) * [set_default_embedding_model(alias) and get_default_embedding_model()](https://llm.datasette.io/en/stable/python-api.html#set-default-embedding-model-alias-and-get-default-embedding-model) * [Logging to SQLite](https://llm.datasette.io/en/stable/logging.html) * [Viewing the logs](https://llm.datasette.io/en/stable/logging.html#viewing-the-logs) * [-s/–short mode](https://llm.datasette.io/en/stable/logging.html#s-short-mode) * [Logs for a conversation](https://llm.datasette.io/en/stable/logging.html#logs-for-a-conversation) * [Searching the logs](https://llm.datasette.io/en/stable/logging.html#searching-the-logs) * [Filtering past a specific ID](https://llm.datasette.io/en/stable/logging.html#filtering-past-a-specific-id) * [Filtering by model](https://llm.datasette.io/en/stable/logging.html#filtering-by-model) * [Filtering by prompts that used specific fragments](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-fragments) * [Filtering by prompts that used specific tools](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-tools) * [Browsing data collected using schemas](https://llm.datasette.io/en/stable/logging.html#browsing-data-collected-using-schemas) * [Browsing logs using Datasette](https://llm.datasette.io/en/stable/logging.html#browsing-logs-using-datasette) * [Backing up your database](https://llm.datasette.io/en/stable/logging.html#backing-up-your-database) * [SQL schema](https://llm.datasette.io/en/stable/logging.html#sql-schema) * [Related tools](https://llm.datasette.io/en/stable/related-tools.html) * [strip-tags](https://llm.datasette.io/en/stable/related-tools.html#strip-tags) * [ttok](https://llm.datasette.io/en/stable/related-tools.html#ttok) * [Symbex](https://llm.datasette.io/en/stable/related-tools.html#symbex) * [CLI reference](https://llm.datasette.io/en/stable/help.html) * [llm –help](https://llm.datasette.io/en/stable/help.html#llm-help) * [llm prompt –help](https://llm.datasette.io/en/stable/help.html#llm-prompt-help) * [llm chat –help](https://llm.datasette.io/en/stable/help.html#llm-chat-help) * [llm keys –help](https://llm.datasette.io/en/stable/help.html#llm-keys-help) * [llm logs –help](https://llm.datasette.io/en/stable/help.html#llm-logs-help) * [llm models –help](https://llm.datasette.io/en/stable/help.html#llm-models-help) * [llm templates –help](https://llm.datasette.io/en/stable/help.html#llm-templates-help) * [llm schemas –help](https://llm.datasette.io/en/stable/help.html#llm-schemas-help) * [llm tools –help](https://llm.datasette.io/en/stable/help.html#llm-tools-help) * [llm aliases –help](https://llm.datasette.io/en/stable/help.html#llm-aliases-help) * [llm fragments –help](https://llm.datasette.io/en/stable/help.html#llm-fragments-help) * [llm plugins –help](https://llm.datasette.io/en/stable/help.html#llm-plugins-help) * [llm install –help](https://llm.datasette.io/en/stable/help.html#llm-install-help) * [llm uninstall –help](https://llm.datasette.io/en/stable/help.html#llm-uninstall-help) * [llm embed –help](https://llm.datasette.io/en/stable/help.html#llm-embed-help) * [llm embed-multi –help](https://llm.datasette.io/en/stable/help.html#llm-embed-multi-help) * [llm similar –help](https://llm.datasette.io/en/stable/help.html#llm-similar-help) * [llm embed-models –help](https://llm.datasette.io/en/stable/help.html#llm-embed-models-help) * [llm collections –help](https://llm.datasette.io/en/stable/help.html#llm-collections-help) * [llm openai –help](https://llm.datasette.io/en/stable/help.html#llm-openai-help) * [Contributing](https://llm.datasette.io/en/stable/contributing.html) * [Updating recorded HTTP API interactions and associated snapshots](https://llm.datasette.io/en/stable/contributing.html#updating-recorded-http-api-interactions-and-associated-snapshots) * [Debugging tricks](https://llm.datasette.io/en/stable/contributing.html#debugging-tricks) * [Documentation](https://llm.datasette.io/en/stable/contributing.html#documentation) * [Release process](https://llm.datasette.io/en/stable/contributing.html#release-process) * [Changelog](https://llm.datasette.io/en/stable/changelog.html) ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1754975852.909846 llm-0.27.1/llm/0000755000175100001660000000000015046547155012570 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/__init__.py0000644000175100001660000003472115046547145014707 0ustar00runnerdockerfrom .hookspecs import hookimpl from .errors import ( ModelError, NeedsKeyException, ) from .models import ( AsyncConversation, AsyncKeyModel, AsyncModel, AsyncResponse, Attachment, CancelToolCall, Conversation, EmbeddingModel, EmbeddingModelWithAliases, KeyModel, Model, ModelWithAliases, Options, Prompt, Response, Tool, Toolbox, ToolCall, ToolOutput, ToolResult, ) from .utils import schema_dsl, Fragment from .embeddings import Collection from .templates import Template from .plugins import pm, load_plugins import click from typing import Any, Dict, List, Optional, Callable, Type, Union import inspect import json import os import pathlib import struct __all__ = [ "AsyncConversation", "AsyncKeyModel", "AsyncResponse", "Attachment", "CancelToolCall", "Collection", "Conversation", "Fragment", "get_async_model", "get_key", "get_model", "hookimpl", "KeyModel", "Model", "ModelError", "NeedsKeyException", "Options", "Prompt", "Response", "Template", "Tool", "Toolbox", "ToolCall", "ToolOutput", "ToolResult", "user_dir", "schema_dsl", ] DEFAULT_MODEL = "gpt-4o-mini" def get_plugins(all=False): plugins = [] plugin_to_distinfo = dict(pm.list_plugin_distinfo()) for plugin in pm.get_plugins(): if not all and plugin.__name__.startswith("llm.default_plugins."): continue plugin_info = { "name": plugin.__name__, "hooks": [h.name for h in pm.get_hookcallers(plugin)], } distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version plugin_info["name"] = ( getattr(distinfo, "name", None) or distinfo.project_name ) plugins.append(plugin_info) return plugins def get_models_with_aliases() -> List["ModelWithAliases"]: model_aliases = [] # Include aliases from aliases.json aliases_path = user_dir() / "aliases.json" extra_model_aliases: Dict[str, list] = {} if aliases_path.exists(): configured_aliases = json.loads(aliases_path.read_text()) for alias, model_id in configured_aliases.items(): extra_model_aliases.setdefault(model_id, []).append(alias) def register(model, async_model=None, aliases=None): alias_list = list(aliases or []) if model.model_id in extra_model_aliases: alias_list.extend(extra_model_aliases[model.model_id]) model_aliases.append(ModelWithAliases(model, async_model, alias_list)) load_plugins() pm.hook.register_models(register=register) return model_aliases def _get_loaders(hook_method) -> Dict[str, Callable]: load_plugins() loaders = {} def register(prefix, loader): suffix = 0 prefix_to_try = prefix while prefix_to_try in loaders: suffix += 1 prefix_to_try = f"{prefix}_{suffix}" loaders[prefix_to_try] = loader hook_method(register=register) return loaders def get_template_loaders() -> Dict[str, Callable[[str], Template]]: """Get template loaders registered by plugins.""" return _get_loaders(pm.hook.register_template_loaders) def get_fragment_loaders() -> Dict[ str, Callable[[str], Union[Fragment, Attachment, List[Union[Fragment, Attachment]]]], ]: """Get fragment loaders registered by plugins.""" return _get_loaders(pm.hook.register_fragment_loaders) def get_tools() -> Dict[str, Union[Tool, Type[Toolbox]]]: """Return all tools (llm.Tool and llm.Toolbox) registered by plugins.""" load_plugins() tools: Dict[str, Union[Tool, Type[Toolbox]]] = {} # Variable to track current plugin name current_plugin_name = None def register( tool_or_function: Union[Tool, Type[Toolbox], Callable[..., Any]], name: Optional[str] = None, ) -> None: tool: Union[Tool, Type[Toolbox], None] = None # If it's a Toolbox class, set the plugin field on it if inspect.isclass(tool_or_function): if issubclass(tool_or_function, Toolbox): tool = tool_or_function if current_plugin_name: tool.plugin = current_plugin_name tool.name = name or tool.__name__ else: raise TypeError( "Toolbox classes must inherit from llm.Toolbox, {} does not.".format( tool_or_function.__name__ ) ) # If it's already a Tool instance, use it directly elif isinstance(tool_or_function, Tool): tool = tool_or_function if name: tool.name = name if current_plugin_name: tool.plugin = current_plugin_name # If it's a bare function, wrap it in a Tool else: tool = Tool.function(tool_or_function, name=name) if current_plugin_name: tool.plugin = current_plugin_name # Get the name for the tool/toolbox if tool: # For Toolbox classes, use their name attribute or class name if inspect.isclass(tool) and issubclass(tool, Toolbox): prefix = name or getattr(tool, "name", tool.__name__) or "" else: prefix = name or tool.name or "" suffix = 0 candidate = prefix # Avoid name collisions while candidate in tools: suffix += 1 candidate = f"{prefix}_{suffix}" tools[candidate] = tool # Call each plugin's register_tools hook individually to track current_plugin_name for plugin in pm.get_plugins(): current_plugin_name = pm.get_name(plugin) hook_caller = pm.hook.register_tools plugin_impls = [ impl for impl in hook_caller.get_hookimpls() if impl.plugin is plugin ] for impl in plugin_impls: impl.function(register=register) return tools def get_embedding_models_with_aliases() -> List["EmbeddingModelWithAliases"]: model_aliases = [] # Include aliases from aliases.json aliases_path = user_dir() / "aliases.json" extra_model_aliases: Dict[str, list] = {} if aliases_path.exists(): configured_aliases = json.loads(aliases_path.read_text()) for alias, model_id in configured_aliases.items(): extra_model_aliases.setdefault(model_id, []).append(alias) def register(model, aliases=None): alias_list = list(aliases or []) if model.model_id in extra_model_aliases: alias_list.extend(extra_model_aliases[model.model_id]) model_aliases.append(EmbeddingModelWithAliases(model, alias_list)) load_plugins() pm.hook.register_embedding_models(register=register) return model_aliases def get_embedding_models(): models = [] def register(model, aliases=None): models.append(model) load_plugins() pm.hook.register_embedding_models(register=register) return models def get_embedding_model(name): aliases = get_embedding_model_aliases() try: return aliases[name] except KeyError: raise UnknownModelError("Unknown model: " + str(name)) def get_embedding_model_aliases() -> Dict[str, EmbeddingModel]: model_aliases = {} for model_with_aliases in get_embedding_models_with_aliases(): for alias in model_with_aliases.aliases: model_aliases[alias] = model_with_aliases.model model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model return model_aliases def get_async_model_aliases() -> Dict[str, AsyncModel]: async_model_aliases = {} for model_with_aliases in get_models_with_aliases(): if model_with_aliases.async_model: for alias in model_with_aliases.aliases: async_model_aliases[alias] = model_with_aliases.async_model async_model_aliases[model_with_aliases.model.model_id] = ( model_with_aliases.async_model ) return async_model_aliases def get_model_aliases() -> Dict[str, Model]: model_aliases = {} for model_with_aliases in get_models_with_aliases(): if model_with_aliases.model: for alias in model_with_aliases.aliases: model_aliases[alias] = model_with_aliases.model model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model return model_aliases class UnknownModelError(KeyError): pass def get_models() -> List[Model]: "Get all registered models" models_with_aliases = get_models_with_aliases() return [mwa.model for mwa in models_with_aliases if mwa.model] def get_async_models() -> List[AsyncModel]: "Get all registered async models" models_with_aliases = get_models_with_aliases() return [mwa.async_model for mwa in models_with_aliases if mwa.async_model] def get_async_model(name: Optional[str] = None) -> AsyncModel: "Get an async model by name or alias" aliases = get_async_model_aliases() name = name or get_default_model() try: return aliases[name] except KeyError: # Does a sync model exist? sync_model = None try: sync_model = get_model(name, _skip_async=True) except UnknownModelError: pass if sync_model: raise UnknownModelError("Unknown async model (sync model exists): " + name) else: raise UnknownModelError("Unknown model: " + name) def get_model(name: Optional[str] = None, _skip_async: bool = False) -> Model: "Get a model by name or alias" aliases = get_model_aliases() name = name or get_default_model() try: return aliases[name] except KeyError: # Does an async model exist? if _skip_async: raise UnknownModelError("Unknown model: " + name) async_model = None try: async_model = get_async_model(name) except UnknownModelError: pass if async_model: raise UnknownModelError("Unknown model (async model exists): " + name) else: raise UnknownModelError("Unknown model: " + name) def get_key( explicit_key: Optional[str] = None, key_alias: Optional[str] = None, env_var: Optional[str] = None, *, alias: Optional[str] = None, env: Optional[str] = None, input: Optional[str] = None, ) -> Optional[str]: """ Return an API key based on a hierarchy of potential sources. You should use the keyword arguments, the positional arguments are here purely for backwards-compatibility with older code. :param input: Input provided by the user. This may be the key, or an alias of a key in keys.json. :param alias: The alias used to retrieve the key from the keys.json file. :param env: Name of the environment variable to check for the key as a final fallback. """ if alias: key_alias = alias if env: env_var = env if input: explicit_key = input stored_keys = load_keys() # If user specified an alias, use the key stored for that alias if explicit_key in stored_keys: return stored_keys[explicit_key] if explicit_key: # User specified a key that's not an alias, use that return explicit_key # Stored key over-rides environment variables over-ride the default key if key_alias in stored_keys: return stored_keys[key_alias] # Finally try environment variable if env_var and os.environ.get(env_var): return os.environ[env_var] # Couldn't find it return None def load_keys(): path = user_dir() / "keys.json" if path.exists(): return json.loads(path.read_text()) else: return {} def user_dir(): llm_user_path = os.environ.get("LLM_USER_PATH") if llm_user_path: path = pathlib.Path(llm_user_path) else: path = pathlib.Path(click.get_app_dir("io.datasette.llm")) path.mkdir(exist_ok=True, parents=True) return path def set_alias(alias, model_id_or_alias): """ Set an alias to point to the specified model. """ path = user_dir() / "aliases.json" path.parent.mkdir(parents=True, exist_ok=True) if not path.exists(): path.write_text("{}\n") try: current = json.loads(path.read_text()) except json.decoder.JSONDecodeError: # We're going to write a valid JSON file in a moment: current = {} # Resolve model_id_or_alias to a model_id try: model = get_model(model_id_or_alias) model_id = model.model_id except UnknownModelError: # Try to resolve it to an embedding model try: model = get_embedding_model(model_id_or_alias) model_id = model.model_id except UnknownModelError: # Set the alias to the exact string they provided instead model_id = model_id_or_alias current[alias] = model_id path.write_text(json.dumps(current, indent=4) + "\n") def remove_alias(alias): """ Remove an alias. """ path = user_dir() / "aliases.json" if not path.exists(): raise KeyError("No aliases.json file exists") try: current = json.loads(path.read_text()) except json.decoder.JSONDecodeError: raise KeyError("aliases.json file is not valid JSON") if alias not in current: raise KeyError("No such alias: {}".format(alias)) del current[alias] path.write_text(json.dumps(current, indent=4) + "\n") def encode(values): return struct.pack("<" + "f" * len(values), *values) def decode(binary): return struct.unpack("<" + "f" * (len(binary) // 4), binary) def cosine_similarity(a, b): dot_product = sum(x * y for x, y in zip(a, b)) magnitude_a = sum(x * x for x in a) ** 0.5 magnitude_b = sum(x * x for x in b) ** 0.5 return dot_product / (magnitude_a * magnitude_b) def get_default_model(filename="default_model.txt", default=DEFAULT_MODEL): path = user_dir() / filename if path.exists(): return path.read_text().strip() else: return default def set_default_model(model, filename="default_model.txt"): path = user_dir() / filename if model is None and path.exists(): path.unlink() else: path.write_text(model) def get_default_embedding_model(): return get_default_model("default_embedding_model.txt", None) def set_default_embedding_model(model): set_default_model(model, "default_embedding_model.txt") ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/__main__.py0000644000175100001660000000007315046547145014661 0ustar00runnerdockerfrom .cli import cli if __name__ == "__main__": cli() ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/cli.py0000644000175100001660000036664615046547145013735 0ustar00runnerdockerimport asyncio import click from click_default_group import DefaultGroup from dataclasses import asdict import io import json import os from llm import ( Attachment, AsyncConversation, AsyncKeyModel, AsyncResponse, CancelToolCall, Collection, Conversation, Fragment, Response, Template, Tool, Toolbox, UnknownModelError, KeyModel, encode, get_async_model, get_default_model, get_default_embedding_model, get_embedding_models_with_aliases, get_embedding_model_aliases, get_embedding_model, get_plugins, get_tools, get_fragment_loaders, get_template_loaders, get_model, get_model_aliases, get_models_with_aliases, user_dir, set_alias, set_default_model, set_default_embedding_model, remove_alias, ) from llm.models import _BaseConversation, ChainResponse from .migrations import migrate from .plugins import pm, load_plugins from .utils import ( ensure_fragment, extract_fenced_code_block, find_unused_key, has_plugin_prefix, instantiate_from_spec, make_schema_id, maybe_fenced_code, mimetype_from_path, mimetype_from_string, multi_schema, output_rows_as_json, resolve_schema_input, schema_dsl, schema_summary, token_usage_string, truncate_string, ) import base64 import httpx import inspect import pathlib import pydantic import re import readline from runpy import run_module import shutil import sqlite_utils from sqlite_utils.utils import rows_from_file, Format import sys import textwrap from typing import cast, Dict, Optional, Iterable, List, Union, Tuple, Type, Any import warnings import yaml warnings.simplefilter("ignore", ResourceWarning) DEFAULT_TEMPLATE = "prompt: " class FragmentNotFound(Exception): pass def validate_fragment_alias(ctx, param, value): if not re.match(r"^[a-zA-Z0-9_-]+$", value): raise click.BadParameter("Fragment alias must be alphanumeric") return value def resolve_fragments( db: sqlite_utils.Database, fragments: Iterable[str], allow_attachments: bool = False ) -> List[Union[Fragment, Attachment]]: """ Resolve fragment strings into a mixed of llm.Fragment() and llm.Attachment() objects. """ def _load_by_alias(fragment: str) -> Tuple[Optional[str], Optional[str]]: rows = list( db.query( """ select content, source from fragments left join fragment_aliases on fragments.id = fragment_aliases.fragment_id where alias = :alias or hash = :alias limit 1 """, {"alias": fragment}, ) ) if rows: row = rows[0] return row["content"], row["source"] return None, None # The fragment strings could be URLs or paths or plugin references resolved: List[Union[Fragment, Attachment]] = [] for fragment in fragments: if fragment.startswith("http://") or fragment.startswith("https://"): client = httpx.Client(follow_redirects=True, max_redirects=3) response = client.get(fragment) response.raise_for_status() resolved.append(Fragment(response.text, fragment)) elif fragment == "-": resolved.append(Fragment(sys.stdin.read(), "-")) elif has_plugin_prefix(fragment): prefix, rest = fragment.split(":", 1) loaders = get_fragment_loaders() if prefix not in loaders: raise FragmentNotFound("Unknown fragment prefix: {}".format(prefix)) loader = loaders[prefix] try: result = loader(rest) if not isinstance(result, list): result = [result] if not allow_attachments and any( isinstance(r, Attachment) for r in result ): raise FragmentNotFound( "Fragment loader {} returned a disallowed attachment".format( prefix ) ) resolved.extend(result) except Exception as ex: raise FragmentNotFound( "Could not load fragment {}: {}".format(fragment, ex) ) else: # Try from the DB content, source = _load_by_alias(fragment) if content is not None: resolved.append(Fragment(content, source)) else: # Now try path path = pathlib.Path(fragment) if path.exists(): resolved.append(Fragment(path.read_text(), str(path.resolve()))) else: raise FragmentNotFound(f"Fragment '{fragment}' not found") return resolved def process_fragments_in_chat( db: sqlite_utils.Database, prompt: str ) -> tuple[str, list[Fragment], list[Attachment]]: """ Process any !fragment commands in a chat prompt and return the modified prompt plus resolved fragments and attachments. """ prompt_lines = [] fragments = [] attachments = [] for line in prompt.splitlines(): if line.startswith("!fragment "): try: fragment_strs = line.strip().removeprefix("!fragment ").split() fragments_and_attachments = resolve_fragments( db, fragments=fragment_strs, allow_attachments=True ) fragments += [ fragment for fragment in fragments_and_attachments if isinstance(fragment, Fragment) ] attachments += [ attachment for attachment in fragments_and_attachments if isinstance(attachment, Attachment) ] except FragmentNotFound as ex: raise click.ClickException(str(ex)) else: prompt_lines.append(line) return "\n".join(prompt_lines), fragments, attachments class AttachmentError(Exception): """Exception raised for errors in attachment resolution.""" pass def resolve_attachment(value): """ Resolve an attachment from a string value which could be: - "-" for stdin - A URL - A file path Returns an Attachment object. Raises AttachmentError if the attachment cannot be resolved. """ if value == "-": content = sys.stdin.buffer.read() # Try to guess type mimetype = mimetype_from_string(content) if mimetype is None: raise AttachmentError("Could not determine mimetype of stdin") return Attachment(type=mimetype, path=None, url=None, content=content) if "://" in value: # Confirm URL exists and try to guess type try: response = httpx.head(value) response.raise_for_status() mimetype = response.headers.get("content-type") except httpx.HTTPError as ex: raise AttachmentError(str(ex)) return Attachment(type=mimetype, path=None, url=value, content=None) # Check that the file exists path = pathlib.Path(value) if not path.exists(): raise AttachmentError(f"File {value} does not exist") path = path.resolve() # Try to guess type mimetype = mimetype_from_path(str(path)) if mimetype is None: raise AttachmentError(f"Could not determine mimetype of {value}") return Attachment(type=mimetype, path=str(path), url=None, content=None) class AttachmentType(click.ParamType): name = "attachment" def convert(self, value, param, ctx): try: return resolve_attachment(value) except AttachmentError as e: self.fail(str(e), param, ctx) def resolve_attachment_with_type(value: str, mimetype: str) -> Attachment: if "://" in value: attachment = Attachment(mimetype, None, value, None) elif value == "-": content = sys.stdin.buffer.read() attachment = Attachment(mimetype, None, None, content) else: # Look for file path = pathlib.Path(value) if not path.exists(): raise click.BadParameter(f"File {value} does not exist") path = path.resolve() attachment = Attachment(mimetype, str(path), None, None) return attachment def attachment_types_callback(ctx, param, values) -> List[Attachment]: collected = [] for value, mimetype in values: collected.append(resolve_attachment_with_type(value, mimetype)) return collected def json_validator(object_name): def validator(ctx, param, value): if value is None: return value try: obj = json.loads(value) if not isinstance(obj, dict): raise click.BadParameter(f"{object_name} must be a JSON object") return obj except json.JSONDecodeError: raise click.BadParameter(f"{object_name} must be valid JSON") return validator def schema_option(fn): click.option( "schema_input", "--schema", help="JSON schema, filepath or ID", )(fn) return fn @click.group( cls=DefaultGroup, default="prompt", default_if_no_args=True, context_settings={"help_option_names": ["-h", "--help"]}, ) @click.version_option() def cli(): """ Access Large Language Models from the command-line Documentation: https://llm.datasette.io/ LLM can run models from many different providers. Consult the plugin directory for a list of available models: https://llm.datasette.io/en/stable/plugins/directory.html To get started with OpenAI, obtain an API key from them and: \b $ llm keys set openai Enter key: ... Then execute a prompt like this: llm 'Five outrageous names for a pet pelican' For a full list of prompting options run: llm prompt --help """ @cli.command(name="prompt") @click.argument("prompt", required=False) @click.option("-s", "--system", help="System prompt to use") @click.option("model_id", "-m", "--model", help="Model to use", envvar="LLM_MODEL") @click.option( "-d", "--database", type=click.Path(readable=True, dir_okay=False), help="Path to log database", ) @click.option( "queries", "-q", "--query", multiple=True, help="Use first model matching these strings", ) @click.option( "attachments", "-a", "--attachment", type=AttachmentType(), multiple=True, help="Attachment path or URL or -", ) @click.option( "attachment_types", "--at", "--attachment-type", type=(str, str), multiple=True, callback=attachment_types_callback, help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", ) @click.option( "tools", "-T", "--tool", multiple=True, help="Name of a tool to make available to the model", ) @click.option( "python_tools", "--functions", help="Python code block or file path defining functions to register as tools", multiple=True, ) @click.option( "tools_debug", "--td", "--tools-debug", is_flag=True, help="Show full details of tool executions", envvar="LLM_TOOLS_DEBUG", ) @click.option( "tools_approve", "--ta", "--tools-approve", is_flag=True, help="Manually approve every tool execution", ) @click.option( "chain_limit", "--cl", "--chain-limit", type=int, default=5, help="How many chained tool responses to allow, default 5, set 0 for unlimited", ) @click.option( "options", "-o", "--option", type=(str, str), multiple=True, help="key/value options for the model", ) @schema_option @click.option( "--schema-multi", help="JSON schema to use for multiple results", ) @click.option( "fragments", "-f", "--fragment", multiple=True, help="Fragment (alias, URL, hash or file path) to add to the prompt", ) @click.option( "system_fragments", "--sf", "--system-fragment", multiple=True, help="Fragment to add to system prompt", ) @click.option("-t", "--template", help="Template to use") @click.option( "-p", "--param", multiple=True, type=(str, str), help="Parameters for template", ) @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("-n", "--no-log", is_flag=True, help="Don't log to database") @click.option("--log", is_flag=True, help="Log prompt and response to the database") @click.option( "_continue", "-c", "--continue", is_flag=True, flag_value=-1, help="Continue the most recent conversation.", ) @click.option( "conversation_id", "--cid", "--conversation", help="Continue the conversation with the given ID.", ) @click.option("--key", help="API key to use") @click.option("--save", help="Save prompt with this template name") @click.option("async_", "--async", is_flag=True, help="Run prompt asynchronously") @click.option("-u", "--usage", is_flag=True, help="Show token usage") @click.option("-x", "--extract", is_flag=True, help="Extract first fenced code block") @click.option( "extract_last", "--xl", "--extract-last", is_flag=True, help="Extract last fenced code block", ) def prompt( prompt, system, model_id, database, queries, attachments, attachment_types, tools, python_tools, tools_debug, tools_approve, chain_limit, options, schema_input, schema_multi, fragments, system_fragments, template, param, no_stream, no_log, log, _continue, conversation_id, key, save, async_, usage, extract, extract_last, ): """ Execute a prompt Documentation: https://llm.datasette.io/en/stable/usage.html Examples: \b llm 'Capital of France?' llm 'Capital of France?' -m gpt-4o llm 'Capital of France?' -s 'answer in Spanish' Multi-modal models can be called with attachments like this: \b llm 'Extract text from this image' -a image.jpg llm 'Describe' -a https://static.simonwillison.net/static/2024/pelicans.jpg cat image | llm 'describe image' -a - # With an explicit mimetype: cat image | llm 'describe image' --at - image/jpeg The -x/--extract option returns just the content of the first ``` fenced code block, if one is present. If none are present it returns the full response. \b llm 'JavaScript function for reversing a string' -x """ if log and no_log: raise click.ClickException("--log and --no-log are mutually exclusive") log_path = pathlib.Path(database) if database else logs_db_path() (log_path.parent).mkdir(parents=True, exist_ok=True) db = sqlite_utils.Database(log_path) migrate(db) if queries and not model_id: # Use -q options to find model with shortest model_id matches = [] for model_with_aliases in get_models_with_aliases(): if all(model_with_aliases.matches(q) for q in queries): matches.append(model_with_aliases.model.model_id) if not matches: raise click.ClickException( "No model found matching queries {}".format(", ".join(queries)) ) model_id = min(matches, key=len) if schema_multi: schema_input = schema_multi schema = resolve_schema_input(db, schema_input, load_template) if schema_multi: # Convert that schema into multiple "items" of the same schema schema = multi_schema(schema) model_aliases = get_model_aliases() def read_prompt(): nonlocal prompt, schema # Is there extra prompt available on stdin? stdin_prompt = None if not sys.stdin.isatty(): stdin_prompt = sys.stdin.read() if stdin_prompt: bits = [stdin_prompt] if prompt: bits.append(prompt) prompt = " ".join(bits) if ( prompt is None and not save and sys.stdin.isatty() and not attachments and not attachment_types and not schema and not fragments ): # Hang waiting for input to stdin (unless --save) prompt = sys.stdin.read() return prompt if save: # We are saving their prompt/system/etc to a new template # Fields to save: prompt, system, model - and more in the future disallowed_options = [] for option, var in ( ("--template", template), ("--continue", _continue), ("--cid", conversation_id), ): if var: disallowed_options.append(option) if disallowed_options: raise click.ClickException( "--save cannot be used with {}".format(", ".join(disallowed_options)) ) path = template_dir() / f"{save}.yaml" to_save = {} if model_id: try: to_save["model"] = model_aliases[model_id].model_id except KeyError: raise click.ClickException("'{}' is not a known model".format(model_id)) prompt = read_prompt() if prompt: to_save["prompt"] = prompt if system: to_save["system"] = system if param: to_save["defaults"] = dict(param) if extract: to_save["extract"] = True if extract_last: to_save["extract_last"] = True if schema: to_save["schema_object"] = schema if fragments: to_save["fragments"] = list(fragments) if system_fragments: to_save["system_fragments"] = list(system_fragments) if python_tools: to_save["functions"] = "\n\n".join(python_tools) if tools: to_save["tools"] = list(tools) if attachments: # Only works for attachments with a path or url to_save["attachments"] = [ (a.path or a.url) for a in attachments if (a.path or a.url) ] if attachment_types: to_save["attachment_types"] = [ {"type": a.type, "value": a.path or a.url} for a in attachment_types if (a.path or a.url) ] if options: # Need to validate and convert their types first model = get_model(model_id or get_default_model()) try: options_model = model.Options(**dict(options)) # Use model_dump(mode="json") so Enums become their .value strings to_save["options"] = { k: v for k, v in options_model.model_dump(mode="json").items() if v is not None } except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) path.write_text( yaml.safe_dump( to_save, indent=4, default_flow_style=False, sort_keys=False, ), "utf-8", ) return if template: params = dict(param) # Cannot be used with system try: template_obj = load_template(template) except LoadTemplateError as ex: raise click.ClickException(str(ex)) extract = template_obj.extract extract_last = template_obj.extract_last # Combine with template fragments/system_fragments if template_obj.fragments: fragments = [*template_obj.fragments, *fragments] if template_obj.system_fragments: system_fragments = [*template_obj.system_fragments, *system_fragments] if template_obj.schema_object: schema = template_obj.schema_object if template_obj.tools: tools = [*template_obj.tools, *tools] if template_obj.functions and template_obj._functions_is_trusted: python_tools = [template_obj.functions, *python_tools] input_ = "" if template_obj.options: # Make options mutable (they start as a tuple) options = list(options) # Load any options, provided they were not set using -o already specified_options = dict(options) for option_name, option_value in template_obj.options.items(): if option_name not in specified_options: options.append((option_name, option_value)) if "input" in template_obj.vars(): input_ = read_prompt() try: template_prompt, template_system = template_obj.evaluate(input_, params) if template_prompt: # Combine with user prompt if prompt and "input" not in template_obj.vars(): prompt = template_prompt + "\n" + prompt else: prompt = template_prompt if template_system and not system: system = template_system except Template.MissingVariables as ex: raise click.ClickException(str(ex)) if model_id is None and template_obj.model: model_id = template_obj.model # Merge in any attachments if template_obj.attachments: attachments = [ resolve_attachment(a) for a in template_obj.attachments ] + list(attachments) if template_obj.attachment_types: attachment_types = [ resolve_attachment_with_type(at.value, at.type) for at in template_obj.attachment_types ] + list(attachment_types) if extract or extract_last: no_stream = True conversation = None if conversation_id or _continue: # Load the conversation - loads most recent if no ID provided try: conversation = load_conversation( conversation_id, async_=async_, database=database ) except UnknownModelError as ex: raise click.ClickException(str(ex)) if conversation_tools := _get_conversation_tools(conversation, tools): tools = conversation_tools # Figure out which model we are using if model_id is None: if conversation: model_id = conversation.model.model_id else: model_id = get_default_model() # Now resolve the model try: if async_: model = get_async_model(model_id) else: model = get_model(model_id) except UnknownModelError as ex: raise click.ClickException(ex) if conversation is None and (tools or python_tools): conversation = model.conversation() if conversation: # To ensure it can see the key conversation.model = model # Validate options validated_options = {} if options: # Validate with pydantic try: validated_options = dict( (key, value) for key, value in model.Options(**dict(options)) if value is not None ) except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) # Add on any default model options default_options = get_model_options(model.model_id) for key_, value in default_options.items(): if key_ not in validated_options: validated_options[key_] = value kwargs = {} resolved_attachments = [*attachments, *attachment_types] should_stream = model.can_stream and not no_stream if not should_stream: kwargs["stream"] = False if isinstance(model, (KeyModel, AsyncKeyModel)): kwargs["key"] = key prompt = read_prompt() response = None try: fragments_and_attachments = resolve_fragments( db, fragments, allow_attachments=True ) resolved_fragments = [ fragment for fragment in fragments_and_attachments if isinstance(fragment, Fragment) ] resolved_attachments.extend( attachment for attachment in fragments_and_attachments if isinstance(attachment, Attachment) ) resolved_system_fragments = resolve_fragments(db, system_fragments) except FragmentNotFound as ex: raise click.ClickException(str(ex)) prompt_method = model.prompt if conversation: prompt_method = conversation.prompt tool_implementations = _gather_tools(tools, python_tools) if tool_implementations: prompt_method = conversation.chain kwargs["options"] = validated_options kwargs["chain_limit"] = chain_limit if tools_debug: kwargs["after_call"] = _debug_tool_call if tools_approve: kwargs["before_call"] = _approve_tool_call kwargs["tools"] = tool_implementations else: # Merge in options for the .prompt() methods kwargs.update(validated_options) try: if async_: async def inner(): if should_stream: response = prompt_method( prompt, attachments=resolved_attachments, system=system, schema=schema, fragments=resolved_fragments, system_fragments=resolved_system_fragments, **kwargs, ) async for chunk in response: print(chunk, end="") sys.stdout.flush() print("") else: response = prompt_method( prompt, fragments=resolved_fragments, attachments=resolved_attachments, schema=schema, system=system, system_fragments=resolved_system_fragments, **kwargs, ) text = await response.text() if extract or extract_last: text = ( extract_fenced_code_block(text, last=extract_last) or text ) print(text) return response response = asyncio.run(inner()) else: response = prompt_method( prompt, fragments=resolved_fragments, attachments=resolved_attachments, system=system, schema=schema, system_fragments=resolved_system_fragments, **kwargs, ) if should_stream: for chunk in response: print(chunk, end="") sys.stdout.flush() print("") else: text = response.text() if extract or extract_last: text = extract_fenced_code_block(text, last=extract_last) or text print(text) # List of exceptions that should never be raised in pytest: except (ValueError, NotImplementedError) as ex: raise click.ClickException(str(ex)) except Exception as ex: # All other exceptions should raise in pytest, show to user otherwise if getattr(sys, "_called_from_test", False) or os.environ.get( "LLM_RAISE_ERRORS", None ): raise raise click.ClickException(str(ex)) if usage: if isinstance(response, ChainResponse): responses = response._responses else: responses = [response] for response_object in responses: # Show token usage to stderr in yellow click.echo( click.style( "Token usage: {}".format(response_object.token_usage()), fg="yellow", bold=True, ), err=True, ) # Log responses to the database if (logs_on() or log) and not no_log: # Could be Response, AsyncResponse, ChainResponse, AsyncChainResponse if isinstance(response, AsyncResponse): response = asyncio.run(response.to_sync_response()) # At this point ALL forms should have a log_to_db() method that works: response.log_to_db(db) @cli.command() @click.option("-s", "--system", help="System prompt to use") @click.option("model_id", "-m", "--model", help="Model to use", envvar="LLM_MODEL") @click.option( "_continue", "-c", "--continue", is_flag=True, flag_value=-1, help="Continue the most recent conversation.", ) @click.option( "conversation_id", "--cid", "--conversation", help="Continue the conversation with the given ID.", ) @click.option( "fragments", "-f", "--fragment", multiple=True, help="Fragment (alias, URL, hash or file path) to add to the prompt", ) @click.option( "system_fragments", "--sf", "--system-fragment", multiple=True, help="Fragment to add to system prompt", ) @click.option("-t", "--template", help="Template to use") @click.option( "-p", "--param", multiple=True, type=(str, str), help="Parameters for template", ) @click.option( "options", "-o", "--option", type=(str, str), multiple=True, help="key/value options for the model", ) @click.option( "-d", "--database", type=click.Path(readable=True, dir_okay=False), help="Path to log database", ) @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("--key", help="API key to use") @click.option( "tools", "-T", "--tool", multiple=True, help="Name of a tool to make available to the model", ) @click.option( "python_tools", "--functions", help="Python code block or file path defining functions to register as tools", multiple=True, ) @click.option( "tools_debug", "--td", "--tools-debug", is_flag=True, help="Show full details of tool executions", envvar="LLM_TOOLS_DEBUG", ) @click.option( "tools_approve", "--ta", "--tools-approve", is_flag=True, help="Manually approve every tool execution", ) @click.option( "chain_limit", "--cl", "--chain-limit", type=int, default=5, help="How many chained tool responses to allow, default 5, set 0 for unlimited", ) def chat( system, model_id, _continue, conversation_id, fragments, system_fragments, template, param, options, no_stream, key, database, tools, python_tools, tools_debug, tools_approve, chain_limit, ): """ Hold an ongoing chat with a model. """ # Left and right arrow keys to move cursor: if sys.platform != "win32": readline.parse_and_bind("\\e[D: backward-char") readline.parse_and_bind("\\e[C: forward-char") else: readline.parse_and_bind("bind -x '\\e[D: backward-char'") readline.parse_and_bind("bind -x '\\e[C: forward-char'") log_path = pathlib.Path(database) if database else logs_db_path() (log_path.parent).mkdir(parents=True, exist_ok=True) db = sqlite_utils.Database(log_path) migrate(db) conversation = None if conversation_id or _continue: # Load the conversation - loads most recent if no ID provided try: conversation = load_conversation(conversation_id, database=database) except UnknownModelError as ex: raise click.ClickException(str(ex)) if conversation_tools := _get_conversation_tools(conversation, tools): tools = conversation_tools template_obj = None if template: params = dict(param) try: template_obj = load_template(template) except LoadTemplateError as ex: raise click.ClickException(str(ex)) if model_id is None and template_obj.model: model_id = template_obj.model if template_obj.tools: tools = [*template_obj.tools, *tools] if template_obj.functions and template_obj._functions_is_trusted: python_tools = [template_obj.functions, *python_tools] # Figure out which model we are using if model_id is None: if conversation: model_id = conversation.model.model_id else: model_id = get_default_model() # Now resolve the model try: model = get_model(model_id) except KeyError: raise click.ClickException("'{}' is not a known model".format(model_id)) if conversation is None: # Start a fresh conversation for this chat conversation = Conversation(model=model) else: # Ensure it can see the API key conversation.model = model if tools_debug: conversation.after_call = _debug_tool_call if tools_approve: conversation.before_call = _approve_tool_call # Validate options validated_options = get_model_options(model.model_id) if options: try: validated_options = dict( (key, value) for key, value in model.Options(**dict(options)) if value is not None ) except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) kwargs = {} if validated_options: kwargs["options"] = validated_options tool_functions = _gather_tools(tools, python_tools) if tool_functions: kwargs["chain_limit"] = chain_limit kwargs["tools"] = tool_functions should_stream = model.can_stream and not no_stream if not should_stream: kwargs["stream"] = False if key and isinstance(model, KeyModel): kwargs["key"] = key try: fragments_and_attachments = resolve_fragments( db, fragments, allow_attachments=True ) argument_fragments = [ fragment for fragment in fragments_and_attachments if isinstance(fragment, Fragment) ] argument_attachments = [ attachment for attachment in fragments_and_attachments if isinstance(attachment, Attachment) ] argument_system_fragments = resolve_fragments(db, system_fragments) except FragmentNotFound as ex: raise click.ClickException(str(ex)) click.echo("Chatting with {}".format(model.model_id)) click.echo("Type 'exit' or 'quit' to exit") click.echo("Type '!multi' to enter multiple lines, then '!end' to finish") click.echo("Type '!edit' to open your default editor and modify the prompt") click.echo( "Type '!fragment [ ...]' to insert one or more fragments" ) in_multi = False accumulated = [] accumulated_fragments = [] accumulated_attachments = [] end_token = "!end" while True: prompt = click.prompt("", prompt_suffix="> " if not in_multi else "") fragments = [] attachments = [] if argument_fragments: fragments += argument_fragments # fragments from --fragments will get added to the first message only argument_fragments = [] if argument_attachments: attachments = argument_attachments argument_attachments = [] if prompt.strip().startswith("!multi"): in_multi = True bits = prompt.strip().split() if len(bits) > 1: end_token = "!end {}".format(" ".join(bits[1:])) continue if prompt.strip() == "!edit": edited_prompt = click.edit() if edited_prompt is None: click.echo("Editor closed without saving.", err=True) continue prompt = edited_prompt.strip() if prompt.strip().startswith("!fragment "): prompt, fragments, attachments = process_fragments_in_chat(db, prompt) if in_multi: if prompt.strip() == end_token: prompt = "\n".join(accumulated) fragments = accumulated_fragments attachments = accumulated_attachments in_multi = False accumulated = [] accumulated_fragments = [] accumulated_attachments = [] else: if prompt: accumulated.append(prompt) accumulated_fragments += fragments accumulated_attachments += attachments continue if template_obj: try: # Mirror prompt() logic: only pass input if template uses it uses_input = "input" in template_obj.vars() input_ = prompt if uses_input else "" template_prompt, template_system = template_obj.evaluate(input_, params) except Template.MissingVariables as ex: raise click.ClickException(str(ex)) if template_system and not system: system = template_system if template_prompt: if prompt and not uses_input: prompt = f"{template_prompt}\n{prompt}" else: prompt = template_prompt if prompt.strip() in ("exit", "quit"): break response = conversation.chain( prompt, fragments=[str(fragment) for fragment in fragments], system_fragments=[ str(system_fragment) for system_fragment in argument_system_fragments ], attachments=attachments, system=system, **kwargs, ) # System prompt and system fragments only sent for the first message system = None argument_system_fragments = [] for chunk in response: print(chunk, end="") sys.stdout.flush() response.log_to_db(db) print("") def load_conversation( conversation_id: Optional[str], async_=False, database=None, ) -> Optional[_BaseConversation]: log_path = pathlib.Path(database) if database else logs_db_path() db = sqlite_utils.Database(log_path) migrate(db) if conversation_id is None: # Return the most recent conversation, or None if there are none matches = list(db["conversations"].rows_where(order_by="id desc", limit=1)) if matches: conversation_id = matches[0]["id"] else: return None try: row = cast(sqlite_utils.db.Table, db["conversations"]).get(conversation_id) except sqlite_utils.db.NotFoundError: raise click.ClickException( "No conversation found with id={}".format(conversation_id) ) # Inflate that conversation conversation_class = AsyncConversation if async_ else Conversation response_class = AsyncResponse if async_ else Response conversation = conversation_class.from_row(row) for response in db["responses"].rows_where( "conversation_id = ?", [conversation_id] ): conversation.responses.append(response_class.from_row(db, response)) return conversation @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def keys(): "Manage stored API keys for different models" @keys.command(name="list") def keys_list(): "List names of all stored keys" path = user_dir() / "keys.json" if not path.exists(): click.echo("No keys found") return keys = json.loads(path.read_text()) for key in sorted(keys.keys()): if key != "// Note": click.echo(key) @keys.command(name="path") def keys_path_command(): "Output the path to the keys.json file" click.echo(user_dir() / "keys.json") @keys.command(name="get") @click.argument("name") def keys_get(name): """ Return the value of a stored key Example usage: \b export OPENAI_API_KEY=$(llm keys get openai) """ path = user_dir() / "keys.json" if not path.exists(): raise click.ClickException("No keys found") keys = json.loads(path.read_text()) try: click.echo(keys[name]) except KeyError: raise click.ClickException("No key found with name '{}'".format(name)) @keys.command(name="set") @click.argument("name") @click.option("--value", prompt="Enter key", hide_input=True, help="Value to set") def keys_set(name, value): """ Save a key in the keys.json file Example usage: \b $ llm keys set openai Enter key: ... """ default = {"// Note": "This file stores secret API credentials. Do not share!"} path = user_dir() / "keys.json" path.parent.mkdir(parents=True, exist_ok=True) if not path.exists(): path.write_text(json.dumps(default)) path.chmod(0o600) try: current = json.loads(path.read_text()) except json.decoder.JSONDecodeError: current = default current[name] = value path.write_text(json.dumps(current, indent=2) + "\n") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def logs(): "Tools for exploring logged prompts and responses" @logs.command(name="path") def logs_path(): "Output the path to the logs.db file" click.echo(logs_db_path()) @logs.command(name="status") def logs_status(): "Show current status of database logging" path = logs_db_path() if not path.exists(): click.echo("No log database found at {}".format(path)) return if logs_on(): click.echo("Logging is ON for all prompts".format()) else: click.echo("Logging is OFF".format()) db = sqlite_utils.Database(path) migrate(db) click.echo("Found log database at {}".format(path)) click.echo("Number of conversations logged:\t{}".format(db["conversations"].count)) click.echo("Number of responses logged:\t{}".format(db["responses"].count)) click.echo( "Database file size: \t\t{}".format(_human_readable_size(path.stat().st_size)) ) @logs.command(name="backup") @click.argument("path", type=click.Path(dir_okay=True, writable=True)) def backup(path): "Backup your logs database to this file" logs_path = logs_db_path() path = pathlib.Path(path) db = sqlite_utils.Database(logs_path) try: db.execute("vacuum into ?", [str(path)]) except Exception as ex: raise click.ClickException(str(ex)) click.echo( "Backed up {} to {}".format(_human_readable_size(path.stat().st_size), path) ) @logs.command(name="on") def logs_turn_on(): "Turn on logging for all prompts" path = user_dir() / "logs-off" if path.exists(): path.unlink() @logs.command(name="off") def logs_turn_off(): "Turn off logging for all prompts" path = user_dir() / "logs-off" path.touch() LOGS_COLUMNS = """ responses.id, responses.model, responses.resolved_model, responses.prompt, responses.system, responses.prompt_json, responses.options_json, responses.response, responses.response_json, responses.conversation_id, responses.duration_ms, responses.datetime_utc, responses.input_tokens, responses.output_tokens, responses.token_details, conversations.name as conversation_name, conversations.model as conversation_model, schemas.content as schema_json""" LOGS_SQL = """ select {columns} from responses left join schemas on responses.schema_id = schemas.id left join conversations on responses.conversation_id = conversations.id{extra_where} order by {order_by}{limit} """ LOGS_SQL_SEARCH = """ select {columns} from responses left join schemas on responses.schema_id = schemas.id left join conversations on responses.conversation_id = conversations.id join responses_fts on responses_fts.rowid = responses.rowid where responses_fts match :query{extra_where} order by {order_by}{limit} """ ATTACHMENTS_SQL = """ select response_id, attachments.id, attachments.type, attachments.path, attachments.url, length(attachments.content) as content_length from attachments join prompt_attachments on attachments.id = prompt_attachments.attachment_id where prompt_attachments.response_id in ({}) order by prompt_attachments."order" """ @logs.command(name="list") @click.option( "-n", "--count", type=int, default=None, help="Number of entries to show - defaults to 3, use 0 for all", ) @click.option( "-p", "--path", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", hidden=True, ) @click.option( "-d", "--database", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", ) @click.option("-m", "--model", help="Filter by model or model alias") @click.option("-q", "--query", help="Search for logs matching this string") @click.option( "fragments", "--fragment", "-f", help="Filter for prompts using these fragments", multiple=True, ) @click.option( "tools", "-T", "--tool", multiple=True, help="Filter for prompts with results from these tools", ) @click.option( "any_tools", "--tools", is_flag=True, help="Filter for prompts with results from any tools", ) @schema_option @click.option( "--schema-multi", help="JSON schema used for multiple results", ) @click.option( "-l", "--latest", is_flag=True, help="Return latest results matching search query" ) @click.option( "--data", is_flag=True, help="Output newline-delimited JSON data for schema" ) @click.option("--data-array", is_flag=True, help="Output JSON array of data for schema") @click.option("--data-key", help="Return JSON objects from array in this key") @click.option( "--data-ids", is_flag=True, help="Attach corresponding IDs to JSON objects" ) @click.option("-t", "--truncate", is_flag=True, help="Truncate long strings in output") @click.option( "-s", "--short", is_flag=True, help="Shorter YAML output with truncated prompts" ) @click.option("-u", "--usage", is_flag=True, help="Include token usage") @click.option("-r", "--response", is_flag=True, help="Just output the last response") @click.option("-x", "--extract", is_flag=True, help="Extract first fenced code block") @click.option( "extract_last", "--xl", "--extract-last", is_flag=True, help="Extract last fenced code block", ) @click.option( "current_conversation", "-c", "--current", is_flag=True, flag_value=-1, help="Show logs from the current conversation", ) @click.option( "conversation_id", "--cid", "--conversation", help="Show logs for this conversation ID", ) @click.option("--id-gt", help="Return responses with ID > this") @click.option("--id-gte", help="Return responses with ID >= this") @click.option( "json_output", "--json", is_flag=True, help="Output logs as JSON", ) @click.option( "--expand", "-e", is_flag=True, help="Expand fragments to show their content", ) def logs_list( count, path, database, model, query, fragments, tools, any_tools, schema_input, schema_multi, latest, data, data_array, data_key, data_ids, truncate, short, usage, response, extract, extract_last, current_conversation, conversation_id, id_gt, id_gte, json_output, expand, ): "Show logged prompts and their responses" if database and not path: path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): raise click.ClickException("No log database found at {}".format(path)) db = sqlite_utils.Database(path) migrate(db) if schema_multi: schema_input = schema_multi schema = resolve_schema_input(db, schema_input, load_template) if schema_multi: schema = multi_schema(schema) if short and (json_output or response): invalid = " or ".join( [ flag[0] for flag in (("--json", json_output), ("--response", response)) if flag[1] ] ) raise click.ClickException("Cannot use --short and {} together".format(invalid)) if response and not current_conversation and not conversation_id: current_conversation = True if current_conversation: try: conversation_id = next( db.query( "select conversation_id from responses order by id desc limit 1" ) )["conversation_id"] except StopIteration: # No conversations yet raise click.ClickException("No conversations found") # For --conversation set limit 0, if not explicitly set if count is None: if conversation_id: count = 0 else: count = 3 model_id = None if model: # Resolve alias, if any try: model_id = get_model(model).model_id except UnknownModelError: # Maybe they uninstalled a model, use the -m option as-is model_id = model sql = LOGS_SQL order_by = "responses.id desc" if query: sql = LOGS_SQL_SEARCH if not latest: order_by = "responses_fts.rank desc" limit = "" if count is not None and count > 0: limit = " limit {}".format(count) sql_format = { "limit": limit, "columns": LOGS_COLUMNS, "extra_where": "", "order_by": order_by, } where_bits = [] sql_params = { "model": model_id, "query": query, "conversation_id": conversation_id, "id_gt": id_gt, "id_gte": id_gte, } if model_id: where_bits.append("responses.model = :model") if conversation_id: where_bits.append("responses.conversation_id = :conversation_id") if id_gt: where_bits.append("responses.id > :id_gt") if id_gte: where_bits.append("responses.id >= :id_gte") if fragments: # Resolve the fragments to their hashes fragment_hashes = [ fragment.id() for fragment in resolve_fragments(db, fragments) ] exists_clauses = [] for i, fragment_hash in enumerate(fragment_hashes): exists_clause = f""" exists ( select 1 from prompt_fragments where prompt_fragments.response_id = responses.id and prompt_fragments.fragment_id in ( select fragments.id from fragments where hash = :f{i} ) union select 1 from system_fragments where system_fragments.response_id = responses.id and system_fragments.fragment_id in ( select fragments.id from fragments where hash = :f{i} ) ) """ exists_clauses.append(exists_clause) sql_params["f{}".format(i)] = fragment_hash where_bits.append(" and ".join(exists_clauses)) if any_tools: # Any response that involved at least one tool result where_bits.append( """ exists ( select 1 from tool_results where tool_results.response_id = responses.id ) """ ) if tools: tools_by_name = get_tools() # Filter responses by tools (must have ALL of the named tools, including plugin) tool_clauses = [] for i, tool_name in enumerate(tools): try: plugin_name = tools_by_name[tool_name].plugin except KeyError: raise click.ClickException(f"Unknown tool: {tool_name}") tool_clauses.append( f""" exists ( select 1 from tool_results join tools on tools.id = tool_results.tool_id where tool_results.response_id = responses.id and tools.name = :tool{i} and tools.plugin = :plugin{i} ) """ ) sql_params[f"tool{i}"] = tool_name sql_params[f"plugin{i}"] = plugin_name # AND means “must have all” — use OR instead if you want “any of” where_bits.append(" and ".join(tool_clauses)) schema_id = None if schema: schema_id = make_schema_id(schema)[0] where_bits.append("responses.schema_id = :schema_id") sql_params["schema_id"] = schema_id if where_bits: where_ = " and " if query else " where " sql_format["extra_where"] = where_ + " and ".join(where_bits) final_sql = sql.format(**sql_format) rows = list(db.query(final_sql, sql_params)) # Reverse the order - we do this because we 'order by id desc limit 3' to get the # 3 most recent results, but we still want to display them in chronological order # ... except for searches where we don't do this if not query and not data: rows.reverse() # Fetch any attachments ids = [row["id"] for row in rows] attachments = list(db.query(ATTACHMENTS_SQL.format(",".join("?" * len(ids))), ids)) attachments_by_id = {} for attachment in attachments: attachments_by_id.setdefault(attachment["response_id"], []).append(attachment) FRAGMENTS_SQL = """ select {table}.response_id, fragments.hash, fragments.id as fragment_id, fragments.content, ( select json_group_array(fragment_aliases.alias) from fragment_aliases where fragment_aliases.fragment_id = fragments.id ) as aliases from {table} join fragments on {table}.fragment_id = fragments.id where {table}.response_id in ({placeholders}) order by {table}."order" """ # Fetch any prompt or system prompt fragments prompt_fragments_by_id = {} system_fragments_by_id = {} for table, dictionary in ( ("prompt_fragments", prompt_fragments_by_id), ("system_fragments", system_fragments_by_id), ): for fragment in db.query( FRAGMENTS_SQL.format(placeholders=",".join("?" * len(ids)), table=table), ids, ): dictionary.setdefault(fragment["response_id"], []).append(fragment) if data or data_array or data_key or data_ids: # Special case for --data to output valid JSON to_output = [] for row in rows: response = row["response"] or "" try: decoded = json.loads(response) new_items = [] if ( isinstance(decoded, dict) and (data_key in decoded) and all(isinstance(item, dict) for item in decoded[data_key]) ): for item in decoded[data_key]: new_items.append(item) else: new_items.append(decoded) if data_ids: for item in new_items: item[find_unused_key(item, "response_id")] = row["id"] item[find_unused_key(item, "conversation_id")] = row["id"] to_output.extend(new_items) except ValueError: pass for line in output_rows_as_json(to_output, nl=not data_array, compact=True): click.echo(line) return # Tool usage information TOOLS_SQL = """ SELECT responses.id, -- Tools related to this response COALESCE( (SELECT json_group_array(json_object( 'id', t.id, 'hash', t.hash, 'name', t.name, 'description', t.description, 'input_schema', json(t.input_schema) )) FROM tools t JOIN tool_responses tr ON t.id = tr.tool_id WHERE tr.response_id = responses.id ), '[]' ) AS tools, -- Tool calls for this response COALESCE( (SELECT json_group_array(json_object( 'id', tc.id, 'tool_id', tc.tool_id, 'name', tc.name, 'arguments', json(tc.arguments), 'tool_call_id', tc.tool_call_id )) FROM tool_calls tc WHERE tc.response_id = responses.id ), '[]' ) AS tool_calls, -- Tool results for this response COALESCE( (SELECT json_group_array(json_object( 'id', tr.id, 'tool_id', tr.tool_id, 'name', tr.name, 'output', tr.output, 'tool_call_id', tr.tool_call_id, 'exception', tr.exception, 'attachments', COALESCE( (SELECT json_group_array(json_object( 'id', a.id, 'type', a.type, 'path', a.path, 'url', a.url, 'content', a.content )) FROM tool_results_attachments tra JOIN attachments a ON tra.attachment_id = a.id WHERE tra.tool_result_id = tr.id ), '[]' ) )) FROM tool_results tr WHERE tr.response_id = responses.id ), '[]' ) AS tool_results FROM responses where id in ({placeholders}) """ tool_info_by_id = { row["id"]: { "tools": json.loads(row["tools"]), "tool_calls": json.loads(row["tool_calls"]), "tool_results": json.loads(row["tool_results"]), } for row in db.query( TOOLS_SQL.format(placeholders=",".join("?" * len(ids))), ids ) } for row in rows: if truncate: row["prompt"] = truncate_string(row["prompt"] or "") row["response"] = truncate_string(row["response"] or "") # Add prompt and system fragments for key in ("prompt_fragments", "system_fragments"): row[key] = [ { "hash": fragment["hash"], "content": ( fragment["content"] if expand else truncate_string(fragment["content"]) ), "aliases": json.loads(fragment["aliases"]), } for fragment in ( prompt_fragments_by_id.get(row["id"], []) if key == "prompt_fragments" else system_fragments_by_id.get(row["id"], []) ) ] # Either decode or remove all JSON keys keys = list(row.keys()) for key in keys: if key.endswith("_json") and row[key] is not None: if truncate: del row[key] else: row[key] = json.loads(row[key]) row.update(tool_info_by_id[row["id"]]) output = None if json_output: # Output as JSON if requested for row in rows: row["attachments"] = [ {k: v for k, v in attachment.items() if k != "response_id"} for attachment in attachments_by_id.get(row["id"], []) ] output = json.dumps(list(rows), indent=2) elif extract or extract_last: # Extract and return first code block for row in rows: output = extract_fenced_code_block(row["response"], last=extract_last) if output is not None: break elif response: # Just output the last response if rows: output = rows[-1]["response"] if output is not None: click.echo(output) else: # Output neatly formatted human-readable logs def _display_fragments(fragments, title): if not fragments: return if not expand: content = "\n".join( ["- {}".format(fragment["hash"]) for fragment in fragments] ) else: #
for each one bits = [] for fragment in fragments: bits.append( "
{}\n{}\n
".format( fragment["hash"], maybe_fenced_code(fragment["content"]) ) ) content = "\n".join(bits) click.echo(f"\n### {title}\n\n{content}") current_system = None should_show_conversation = True for row in rows: if short: system = truncate_string( row["system"] or "", 120, normalize_whitespace=True ) prompt = truncate_string( row["prompt"] or "", 120, normalize_whitespace=True, keep_end=True ) cid = row["conversation_id"] attachments = attachments_by_id.get(row["id"]) obj = { "model": row["model"], "datetime": row["datetime_utc"].split(".")[0], "conversation": cid, } if row["tool_calls"]: obj["tool_calls"] = [ "{}({})".format( tool_call["name"], json.dumps(tool_call["arguments"]) ) for tool_call in row["tool_calls"] ] if row["tool_results"]: obj["tool_results"] = [ "{}: {}".format( tool_result["name"], truncate_string(tool_result["output"]) ) for tool_result in row["tool_results"] ] if system: obj["system"] = system if prompt: obj["prompt"] = prompt if attachments: items = [] for attachment in attachments: details = {"type": attachment["type"]} if attachment.get("path"): details["path"] = attachment["path"] if attachment.get("url"): details["url"] = attachment["url"] items.append(details) obj["attachments"] = items for key in ("prompt_fragments", "system_fragments"): obj[key] = [fragment["hash"] for fragment in row[key]] if usage and (row["input_tokens"] or row["output_tokens"]): usage_details = { "input": row["input_tokens"], "output": row["output_tokens"], } if row["token_details"]: usage_details["details"] = json.loads(row["token_details"]) obj["usage"] = usage_details click.echo(yaml.dump([obj], sort_keys=False).strip()) continue # Not short, output Markdown click.echo( "# {}{}\n{}".format( row["datetime_utc"].split(".")[0], ( " conversation: {} id: {}".format( row["conversation_id"], row["id"] ) if should_show_conversation else "" ), ( ( "\nModel: **{}**{}\n".format( row["model"], ( " (resolved: **{}**)".format(row["resolved_model"]) if row["resolved_model"] else "" ), ) ) if should_show_conversation else "" ), ) ) # In conversation log mode only show it for the first one if conversation_id: should_show_conversation = False click.echo("## Prompt\n\n{}".format(row["prompt"] or "-- none --")) _display_fragments(row["prompt_fragments"], "Prompt fragments") if row["system"] != current_system: if row["system"] is not None: click.echo("\n## System\n\n{}".format(row["system"])) current_system = row["system"] _display_fragments(row["system_fragments"], "System fragments") if row["schema_json"]: click.echo( "\n## Schema\n\n```json\n{}\n```".format( json.dumps(row["schema_json"], indent=2) ) ) # Show tool calls and results if row["tools"]: click.echo("\n### Tools\n") for tool in row["tools"]: click.echo( "- **{}**: `{}`
\n {}
\n Arguments: {}".format( tool["name"], tool["hash"], tool["description"], json.dumps(tool["input_schema"]["properties"]), ) ) if row["tool_results"]: click.echo("\n### Tool results\n") for tool_result in row["tool_results"]: attachments = "" for attachment in tool_result["attachments"]: desc = "" if attachment.get("type"): desc += attachment["type"] + ": " if attachment.get("path"): desc += attachment["path"] elif attachment.get("url"): desc += attachment["url"] elif attachment.get("content"): desc += f"<{attachment['content_length']:,} bytes>" attachments += "\n - {}".format(desc) click.echo( "- **{}**: `{}`
\n{}{}{}".format( tool_result["name"], tool_result["tool_call_id"], textwrap.indent(tool_result["output"], " "), ( "
\n **Error**: {}\n".format( tool_result["exception"] ) if tool_result["exception"] else "" ), attachments, ) ) attachments = attachments_by_id.get(row["id"]) if attachments: click.echo("\n### Attachments\n") for i, attachment in enumerate(attachments, 1): if attachment["path"]: path = attachment["path"] click.echo( "{}. **{}**: `{}`".format(i, attachment["type"], path) ) elif attachment["url"]: click.echo( "{}. **{}**: {}".format( i, attachment["type"], attachment["url"] ) ) elif attachment["content_length"]: click.echo( "{}. **{}**: `<{} bytes>`".format( i, attachment["type"], f"{attachment['content_length']:,}", ) ) # If a schema was provided and the row is valid JSON, pretty print and syntax highlight it response = row["response"] if row["schema_json"]: try: parsed = json.loads(response) response = "```json\n{}\n```".format(json.dumps(parsed, indent=2)) except ValueError: pass click.echo("\n## Response\n") if row["tool_calls"]: click.echo("### Tool calls\n") for tool_call in row["tool_calls"]: click.echo( "- **{}**: `{}`
\n Arguments: {}".format( tool_call["name"], tool_call["tool_call_id"], json.dumps(tool_call["arguments"]), ) ) click.echo("") if response: click.echo("{}\n".format(response)) if usage: token_usage = token_usage_string( row["input_tokens"], row["output_tokens"], json.loads(row["token_details"]) if row["token_details"] else None, ) if token_usage: click.echo("## Token usage\n\n{}\n".format(token_usage)) @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def models(): "Manage available models" _type_lookup = { "number": "float", "integer": "int", "string": "str", "object": "dict", } @models.command(name="list") @click.option( "--options", is_flag=True, help="Show options for each model, if available" ) @click.option("async_", "--async", is_flag=True, help="List async models") @click.option("--schemas", is_flag=True, help="List models that support schemas") @click.option("--tools", is_flag=True, help="List models that support tools") @click.option( "-q", "--query", multiple=True, help="Search for models matching these strings", ) @click.option("model_ids", "-m", "--model", help="Specific model IDs", multiple=True) def models_list(options, async_, schemas, tools, query, model_ids): "List available models" models_that_have_shown_options = set() for model_with_aliases in get_models_with_aliases(): if async_ and not model_with_aliases.async_model: continue if query: # Only show models where every provided query string matches if not all(model_with_aliases.matches(q) for q in query): continue if model_ids: ids_and_aliases = set( [model_with_aliases.model.model_id] + model_with_aliases.aliases ) if not ids_and_aliases.intersection(model_ids): continue if schemas and not model_with_aliases.model.supports_schema: continue if tools and not model_with_aliases.model.supports_tools: continue extra_info = [] if model_with_aliases.aliases: extra_info.append( "aliases: {}".format(", ".join(model_with_aliases.aliases)) ) model = ( model_with_aliases.model if not async_ else model_with_aliases.async_model ) output = str(model) if extra_info: output += " ({})".format(", ".join(extra_info)) if options and model.Options.model_json_schema()["properties"]: output += "\n Options:" for name, field in model.Options.model_json_schema()["properties"].items(): any_of = field.get("anyOf") if any_of is None: any_of = [{"type": field.get("type", "str")}] types = ", ".join( [ _type_lookup.get(item.get("type"), item.get("type", "str")) for item in any_of if item.get("type") != "null" ] ) bits = ["\n ", name, ": ", types] description = field.get("description", "") if description and ( model.__class__ not in models_that_have_shown_options ): wrapped = textwrap.wrap(description, 70) bits.append("\n ") bits.extend("\n ".join(wrapped)) output += "".join(bits) models_that_have_shown_options.add(model.__class__) if options and model.attachment_types: attachment_types = ", ".join(sorted(model.attachment_types)) wrapper = textwrap.TextWrapper( width=min(max(shutil.get_terminal_size().columns, 30), 70), initial_indent=" ", subsequent_indent=" ", ) output += "\n Attachment types:\n{}".format(wrapper.fill(attachment_types)) features = ( [] + (["streaming"] if model.can_stream else []) + (["schemas"] if model.supports_schema else []) + (["tools"] if model.supports_tools else []) + (["async"] if model_with_aliases.async_model else []) ) if options and features: output += "\n Features:\n{}".format( "\n".join(" - {}".format(feature) for feature in features) ) if options and hasattr(model, "needs_key") and model.needs_key: output += "\n Keys:" if hasattr(model, "needs_key") and model.needs_key: output += "\n key: {}".format(model.needs_key) if hasattr(model, "key_env_var") and model.key_env_var: output += "\n env_var: {}".format(model.key_env_var) click.echo(output) if not query and not options and not schemas and not model_ids: click.echo(f"Default: {get_default_model()}") @models.command(name="default") @click.argument("model", required=False) def models_default(model): "Show or set the default model" if not model: click.echo(get_default_model()) return # Validate it is a known model try: model = get_model(model) set_default_model(model.model_id) except KeyError: raise click.ClickException("Unknown model: {}".format(model)) @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def templates(): "Manage stored prompt templates" @templates.command(name="list") def templates_list(): "List available prompt templates" path = template_dir() pairs = [] for file in path.glob("*.yaml"): name = file.stem try: template = load_template(name) except LoadTemplateError: # Skip invalid templates continue text = [] if template.system: text.append(f"system: {template.system}") if template.prompt: text.append(f" prompt: {template.prompt}") else: text = [template.prompt if template.prompt else ""] pairs.append((name, "".join(text).replace("\n", " "))) try: max_name_len = max(len(p[0]) for p in pairs) except ValueError: return else: fmt = "{name:<" + str(max_name_len) + "} : {prompt}" for name, prompt in sorted(pairs): text = fmt.format(name=name, prompt=prompt) click.echo(display_truncated(text)) @templates.command(name="show") @click.argument("name") def templates_show(name): "Show the specified prompt template" try: template = load_template(name) except LoadTemplateError: raise click.ClickException(f"Template '{name}' not found or invalid") click.echo( yaml.dump( dict((k, v) for k, v in template.model_dump().items() if v is not None), indent=4, default_flow_style=False, ) ) @templates.command(name="edit") @click.argument("name") def templates_edit(name): "Edit the specified prompt template using the default $EDITOR" # First ensure it exists path = template_dir() / f"{name}.yaml" if not path.exists(): path.write_text(DEFAULT_TEMPLATE, "utf-8") click.edit(filename=str(path)) # Validate that template load_template(name) @templates.command(name="path") def templates_path(): "Output the path to the templates directory" click.echo(template_dir()) @templates.command(name="loaders") def templates_loaders(): "Show template loaders registered by plugins" found = False for prefix, loader in get_template_loaders().items(): found = True docs = "Undocumented" if loader.__doc__: docs = textwrap.dedent(loader.__doc__).strip() click.echo(f"{prefix}:") click.echo(textwrap.indent(docs, " ")) if not found: click.echo("No template loaders found") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def schemas(): "Manage stored schemas" @schemas.command(name="list") @click.option( "-p", "--path", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", hidden=True, ) @click.option( "-d", "--database", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", ) @click.option( "queries", "-q", "--query", multiple=True, help="Search for schemas matching this string", ) @click.option("--full", is_flag=True, help="Output full schema contents") @click.option("json_", "--json", is_flag=True, help="Output as JSON") @click.option("nl", "--nl", is_flag=True, help="Output as newline-delimited JSON") def schemas_list(path, database, queries, full, json_, nl): "List stored schemas" if database and not path: path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): raise click.ClickException("No log database found at {}".format(path)) db = sqlite_utils.Database(path) migrate(db) params = [] where_sql = "" if queries: where_bits = ["schemas.content like ?" for _ in queries] where_sql += " where {}".format(" and ".join(where_bits)) params.extend("%{}%".format(q) for q in queries) sql = """ select schemas.id, schemas.content, max(responses.datetime_utc) as recently_used, count(*) as times_used from schemas join responses on responses.schema_id = schemas.id {} group by responses.schema_id order by recently_used """.format( where_sql ) rows = db.query(sql, params) if json_ or nl: for line in output_rows_as_json(rows, json_cols={"content"}, nl=nl): click.echo(line) return for row in rows: click.echo("- id: {}".format(row["id"])) if full: click.echo( " schema: |\n{}".format( textwrap.indent( json.dumps(json.loads(row["content"]), indent=2), " " ) ) ) else: click.echo( " summary: |\n {}".format( schema_summary(json.loads(row["content"])) ) ) click.echo( " usage: |\n {} time{}, most recently {}".format( row["times_used"], "s" if row["times_used"] != 1 else "", row["recently_used"], ) ) @schemas.command(name="show") @click.argument("schema_id") @click.option( "-p", "--path", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", hidden=True, ) @click.option( "-d", "--database", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", ) def schemas_show(schema_id, path, database): "Show a stored schema" if database and not path: path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): raise click.ClickException("No log database found at {}".format(path)) db = sqlite_utils.Database(path) migrate(db) try: row = db["schemas"].get(schema_id) except sqlite_utils.db.NotFoundError: raise click.ClickException("Invalid schema ID") click.echo(json.dumps(json.loads(row["content"]), indent=2)) @schemas.command(name="dsl") @click.argument("input") @click.option("--multi", is_flag=True, help="Wrap in an array") def schemas_dsl_debug(input, multi): """ Convert LLM's schema DSL to a JSON schema \b llm schema dsl 'name, age int, bio: their bio' """ schema = schema_dsl(input, multi) click.echo(json.dumps(schema, indent=2)) @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def tools(): "Manage tools that can be made available to LLMs" @tools.command(name="list") @click.argument("tool_defs", nargs=-1) @click.option("json_", "--json", is_flag=True, help="Output as JSON") @click.option( "python_tools", "--functions", help="Python code block or file path defining functions to register as tools", multiple=True, ) def tools_list(tool_defs, json_, python_tools): "List available tools that have been provided by plugins" def introspect_tools(toolbox_class): methods = [] for tool in toolbox_class.method_tools(): methods.append( { "name": tool.name, "description": tool.description, "arguments": tool.input_schema, "implementation": tool.implementation, } ) return methods if tool_defs: tools = {} for tool in _gather_tools(tool_defs, python_tools): if hasattr(tool, "name"): tools[tool.name] = tool else: tools[tool.__class__.__name__] = tool else: tools = get_tools() if python_tools: for code_or_path in python_tools: for tool in _tools_from_code(code_or_path): tools[tool.name] = tool output_tools = [] output_toolboxes = [] tool_objects = [] toolbox_objects = [] for name, tool in sorted(tools.items()): if isinstance(tool, Tool): tool_objects.append(tool) output_tools.append( { "name": name, "description": tool.description, "arguments": tool.input_schema, "plugin": tool.plugin, } ) else: toolbox_objects.append(tool) output_toolboxes.append( { "name": name, "tools": [ { "name": tool["name"], "description": tool["description"], "arguments": tool["arguments"], } for tool in introspect_tools(tool) ], } ) if json_: click.echo( json.dumps( {"tools": output_tools, "toolboxes": output_toolboxes}, indent=2, ) ) else: for tool in tool_objects: sig = "()" if tool.implementation: sig = str(inspect.signature(tool.implementation)) click.echo( "{}{}{}\n".format( tool.name, sig, " (plugin: {})".format(tool.plugin) if tool.plugin else "", ) ) if tool.description: click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") for toolbox in toolbox_objects: click.echo(toolbox.name + ":\n") for tool in toolbox.method_tools(): sig = ( str(inspect.signature(tool.implementation)) .replace("(self, ", "(") .replace("(self)", "()") ) click.echo( " {}{}\n".format( tool.name, sig, ) ) if tool.description: click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def aliases(): "Manage model aliases" @aliases.command(name="list") @click.option("json_", "--json", is_flag=True, help="Output as JSON") def aliases_list(json_): "List current aliases" to_output = [] for alias, model in get_model_aliases().items(): if alias != model.model_id: to_output.append((alias, model.model_id, "")) for alias, embedding_model in get_embedding_model_aliases().items(): if alias != embedding_model.model_id: to_output.append((alias, embedding_model.model_id, "embedding")) if json_: click.echo( json.dumps({key: value for key, value, type_ in to_output}, indent=4) ) return max_alias_length = max(len(a) for a, _, _ in to_output) fmt = "{alias:<" + str(max_alias_length) + "} : {model_id}{type_}" for alias, model_id, type_ in to_output: click.echo( fmt.format( alias=alias, model_id=model_id, type_=f" ({type_})" if type_ else "" ) ) @aliases.command(name="set") @click.argument("alias") @click.argument("model_id", required=False) @click.option( "-q", "--query", multiple=True, help="Set alias for model matching these strings", ) def aliases_set(alias, model_id, query): """ Set an alias for a model Example usage: \b llm aliases set mini gpt-4o-mini Alternatively you can omit the model ID and specify one or more -q options. The first model matching all of those query strings will be used. \b llm aliases set mini -q 4o -q mini """ if not model_id: if not query: raise click.ClickException( "You must provide a model_id or at least one -q option" ) # Search for the first model matching all query strings found = None for model_with_aliases in get_models_with_aliases(): if all(model_with_aliases.matches(q) for q in query): found = model_with_aliases break if not found: raise click.ClickException( "No model found matching query: " + ", ".join(query) ) model_id = found.model.model_id set_alias(alias, model_id) click.echo( f"Alias '{alias}' set to model '{model_id}'", err=True, ) else: set_alias(alias, model_id) @aliases.command(name="remove") @click.argument("alias") def aliases_remove(alias): """ Remove an alias Example usage: \b $ llm aliases remove turbo """ try: remove_alias(alias) except KeyError as ex: raise click.ClickException(ex.args[0]) @aliases.command(name="path") def aliases_path(): "Output the path to the aliases.json file" click.echo(user_dir() / "aliases.json") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def fragments(): """ Manage fragments that are stored in the database Fragments are reusable snippets of text that are shared across multiple prompts. """ @fragments.command(name="list") @click.option( "queries", "-q", "--query", multiple=True, help="Search for fragments matching these strings", ) @click.option("--aliases", is_flag=True, help="Show only fragments with aliases") @click.option("json_", "--json", is_flag=True, help="Output as JSON") def fragments_list(queries, aliases, json_): "List current fragments" db = sqlite_utils.Database(logs_db_path()) migrate(db) params = {} param_count = 0 where_bits = [] if aliases: where_bits.append("fragment_aliases.alias is not null") for q in queries: param_count += 1 p = f"p{param_count}" params[p] = q where_bits.append( f""" (fragments.hash = :{p} or fragment_aliases.alias = :{p} or fragments.source like '%' || :{p} || '%' or fragments.content like '%' || :{p} || '%') """ ) where = "\n and\n ".join(where_bits) if where: where = " where " + where sql = """ select fragments.hash, json_group_array(fragment_aliases.alias) filter ( where fragment_aliases.alias is not null ) as aliases, fragments.datetime_utc, fragments.source, fragments.content from fragments left join fragment_aliases on fragment_aliases.fragment_id = fragments.id {where} group by fragments.id, fragments.hash, fragments.content, fragments.datetime_utc, fragments.source order by fragments.datetime_utc """.format( where=where ) results = list(db.query(sql, params)) for result in results: result["aliases"] = json.loads(result["aliases"]) if json_: click.echo(json.dumps(results, indent=4)) else: yaml.add_representer( str, lambda dumper, data: dumper.represent_scalar( "tag:yaml.org,2002:str", data, style="|" if "\n" in data else None ), ) for result in results: result["content"] = truncate_string(result["content"]) click.echo(yaml.dump([result], sort_keys=False, width=sys.maxsize).strip()) @fragments.command(name="set") @click.argument("alias", callback=validate_fragment_alias) @click.argument("fragment") def fragments_set(alias, fragment): """ Set an alias for a fragment Accepts an alias and a file path, URL, hash or '-' for stdin Example usage: \b llm fragments set mydocs ./docs.md """ db = sqlite_utils.Database(logs_db_path()) migrate(db) try: resolved = resolve_fragments(db, [fragment])[0] except FragmentNotFound as ex: raise click.ClickException(str(ex)) migrate(db) alias_sql = """ insert into fragment_aliases (alias, fragment_id) values (:alias, :fragment_id) on conflict(alias) do update set fragment_id = excluded.fragment_id; """ with db.conn: fragment_id = ensure_fragment(db, resolved) db.conn.execute(alias_sql, {"alias": alias, "fragment_id": fragment_id}) @fragments.command(name="show") @click.argument("alias_or_hash") def fragments_show(alias_or_hash): """ Display the fragment stored under an alias or hash \b llm fragments show mydocs """ db = sqlite_utils.Database(logs_db_path()) migrate(db) try: resolved = resolve_fragments(db, [alias_or_hash])[0] except FragmentNotFound as ex: raise click.ClickException(str(ex)) click.echo(resolved) @fragments.command(name="remove") @click.argument("alias", callback=validate_fragment_alias) def fragments_remove(alias): """ Remove a fragment alias Example usage: \b llm fragments remove docs """ db = sqlite_utils.Database(logs_db_path()) migrate(db) with db.conn: db.conn.execute( "delete from fragment_aliases where alias = :alias", {"alias": alias} ) @fragments.command(name="loaders") def fragments_loaders(): """Show fragment loaders registered by plugins""" from llm import get_fragment_loaders found = False for prefix, loader in get_fragment_loaders().items(): if found: # Extra newline on all after the first click.echo("") found = True docs = "Undocumented" if loader.__doc__: docs = textwrap.dedent(loader.__doc__).strip() click.echo(f"{prefix}:") click.echo(textwrap.indent(docs, " ")) if not found: click.echo("No fragment loaders found") @cli.command(name="plugins") @click.option("--all", help="Include built-in default plugins", is_flag=True) @click.option( "hooks", "--hook", help="Filter for plugins that implement this hook", multiple=True ) def plugins_list(all, hooks): "List installed plugins" plugins = get_plugins(all) hooks = set(hooks) if hooks: plugins = [plugin for plugin in plugins if hooks.intersection(plugin["hooks"])] click.echo(json.dumps(plugins, indent=2)) def display_truncated(text): console_width = shutil.get_terminal_size()[0] if len(text) > console_width: return text[: console_width - 3] + "..." else: return text @cli.command() @click.argument("packages", nargs=-1, required=False) @click.option( "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version" ) @click.option( "-e", "--editable", help="Install a project in editable mode from this path", ) @click.option( "--force-reinstall", is_flag=True, help="Reinstall all packages even if they are already up-to-date", ) @click.option( "--no-cache-dir", is_flag=True, help="Disable the cache", ) @click.option( "--pre", is_flag=True, help="Include pre-release and development versions", ) def install(packages, upgrade, editable, force_reinstall, no_cache_dir, pre): """Install packages from PyPI into the same environment as LLM""" args = ["pip", "install"] if upgrade: args += ["--upgrade"] if editable: args += ["--editable", editable] if force_reinstall: args += ["--force-reinstall"] if no_cache_dir: args += ["--no-cache-dir"] if pre: args += ["--pre"] args += list(packages) sys.argv = args run_module("pip", run_name="__main__") @cli.command() @click.argument("packages", nargs=-1, required=True) @click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") def uninstall(packages, yes): """Uninstall Python packages from the LLM environment""" sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else []) run_module("pip", run_name="__main__") @cli.command() @click.argument("collection", required=False) @click.argument("id", required=False) @click.option( "-i", "--input", type=click.Path(exists=True, readable=True, allow_dash=True), help="File to embed", ) @click.option( "-m", "--model", help="Embedding model to use", envvar="LLM_EMBEDDING_MODEL" ) @click.option("--store", is_flag=True, help="Store the text itself in the database") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", ) @click.option( "-c", "--content", help="Content to embed", ) @click.option("--binary", is_flag=True, help="Treat input as binary data") @click.option( "--metadata", help="JSON object metadata to store", callback=json_validator("metadata"), ) @click.option( "format_", "-f", "--format", type=click.Choice(["json", "blob", "base64", "hex"]), help="Output format", ) def embed( collection, id, input, model, store, database, content, binary, metadata, format_ ): """Embed text and store or return the result""" if collection and not id: raise click.ClickException("Must provide both collection and id") if store and not collection: raise click.ClickException("Must provide collection when using --store") # Lazy load this because we do not need it for -c or -i versions def get_db(): if database: return sqlite_utils.Database(database) else: return sqlite_utils.Database(user_dir() / "embeddings.db") collection_obj = None model_obj = None if collection: db = get_db() if Collection.exists(db, collection): # Load existing collection and use its model collection_obj = Collection(collection, db) model_obj = collection_obj.model() else: # We will create a new one, but that means model is required if not model: model = get_default_embedding_model() if model is None: raise click.ClickException( "You need to specify an embedding model (no default model is set)" ) collection_obj = Collection(collection, db=db, model_id=model) model_obj = collection_obj.model() if model_obj is None: if model is None: model = get_default_embedding_model() try: model_obj = get_embedding_model(model) except UnknownModelError: raise click.ClickException( "You need to specify an embedding model (no default model is set)" ) show_output = True if collection and (format_ is None): show_output = False # Resolve input text if not content: if not input or input == "-": # Read from stdin input_source = sys.stdin.buffer if binary else sys.stdin content = input_source.read() else: mode = "rb" if binary else "r" with open(input, mode) as f: content = f.read() if not content: raise click.ClickException("No content provided") if collection_obj: embedding = collection_obj.embed(id, content, metadata=metadata, store=store) else: embedding = model_obj.embed(content) if show_output: if format_ == "json" or format_ is None: click.echo(json.dumps(embedding)) elif format_ == "blob": click.echo(encode(embedding)) elif format_ == "base64": click.echo(base64.b64encode(encode(embedding)).decode("ascii")) elif format_ == "hex": click.echo(encode(embedding).hex()) @cli.command() @click.argument("collection") @click.argument( "input_path", type=click.Path(exists=True, dir_okay=False, allow_dash=True, readable=True), required=False, ) @click.option( "--format", type=click.Choice(["json", "csv", "tsv", "nl"]), help="Format of input file - defaults to auto-detect", ) @click.option( "--files", type=(click.Path(file_okay=False, dir_okay=True, allow_dash=False), str), multiple=True, help="Embed files in this directory - specify directory and glob pattern", ) @click.option( "encodings", "--encoding", help="Encodings to try when reading --files", multiple=True, ) @click.option("--binary", is_flag=True, help="Treat --files as binary data") @click.option("--sql", help="Read input using this SQL query") @click.option( "--attach", type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), multiple=True, help="Additional databases to attach - specify alias and file path", ) @click.option( "--batch-size", type=int, help="Batch size to use when running embeddings" ) @click.option("--prefix", help="Prefix to add to the IDs", default="") @click.option( "-m", "--model", help="Embedding model to use", envvar="LLM_EMBEDDING_MODEL" ) @click.option( "--prepend", help="Prepend this string to all content before embedding", ) @click.option("--store", is_flag=True, help="Store the text itself in the database") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", ) def embed_multi( collection, input_path, format, files, encodings, binary, sql, attach, batch_size, prefix, model, prepend, store, database, ): """ Store embeddings for multiple strings at once in the specified collection. Input data can come from one of three sources: \b 1. A CSV, TSV, JSON or JSONL file: - CSV/TSV: First column is ID, remaining columns concatenated as content - JSON: Array of objects with "id" field and content fields - JSONL: Newline-delimited JSON objects \b Examples: llm embed-multi docs input.csv cat data.json | llm embed-multi docs - llm embed-multi docs input.json --format json \b 2. A SQL query against a SQLite database: - First column returned is used as ID - Other columns concatenated to form content \b Examples: llm embed-multi docs --sql "SELECT id, title, body FROM posts" llm embed-multi docs --attach blog blog.db --sql "SELECT id, content FROM blog.posts" \b 3. Files in directories matching glob patterns: - Each file becomes one embedding - Relative file paths become IDs \b Examples: llm embed-multi docs --files docs '**/*.md' llm embed-multi images --files photos '*.jpg' --binary llm embed-multi texts --files texts '*.txt' --encoding utf-8 --encoding latin-1 """ if binary and not files: raise click.UsageError("--binary must be used with --files") if binary and encodings: raise click.UsageError("--binary cannot be used with --encoding") if not input_path and not sql and not files: raise click.UsageError("Either --sql or input path or --files is required") if files: if input_path or sql or format: raise click.UsageError( "Cannot use --files with --sql, input path or --format" ) if database: db = sqlite_utils.Database(database) else: db = sqlite_utils.Database(user_dir() / "embeddings.db") for alias, attach_path in attach: db.attach(alias, attach_path) try: collection_obj = Collection( collection, db=db, model_id=model or get_default_embedding_model() ) except ValueError: raise click.ClickException( "You need to specify an embedding model (no default model is set)" ) expected_length = None if files: encodings = encodings or ("utf-8", "latin-1") def count_files(): i = 0 for directory, pattern in files: for path in pathlib.Path(directory).glob(pattern): i += 1 return i def iterate_files(): for directory, pattern in files: p = pathlib.Path(directory) if not p.exists() or not p.is_dir(): # fixes issue/274 - raise error if directory does not exist raise click.UsageError(f"Invalid directory: {directory}") for path in pathlib.Path(directory).glob(pattern): if path.is_dir(): continue # fixed issue/280 - skip directories relative = path.relative_to(directory) content = None if binary: content = path.read_bytes() else: for encoding in encodings: try: content = path.read_text(encoding=encoding) except UnicodeDecodeError: continue if content is None: # Log to stderr click.echo( "Could not decode text in file {}".format(path), err=True, ) else: yield {"id": str(relative), "content": content} expected_length = count_files() rows = iterate_files() elif sql: rows = db.query(sql) count_sql = "select count(*) as c from ({})".format(sql) expected_length = next(db.query(count_sql))["c"] else: def load_rows(fp): return rows_from_file(fp, Format[format.upper()] if format else None)[0] try: if input_path != "-": # Read the file twice - first time is to get a count expected_length = 0 with open(input_path, "rb") as fp: for _ in load_rows(fp): expected_length += 1 rows = load_rows( open(input_path, "rb") if input_path != "-" else io.BufferedReader(sys.stdin.buffer) ) except json.JSONDecodeError as ex: raise click.ClickException(str(ex)) with click.progressbar( rows, label="Embedding", show_percent=True, length=expected_length ) as rows: def tuples() -> Iterable[Tuple[str, Union[bytes, str]]]: for row in rows: values = list(row.values()) id: str = prefix + str(values[0]) content: Optional[Union[bytes, str]] = None if binary: content = cast(bytes, values[1]) else: content = " ".join(v or "" for v in values[1:]) if prepend and isinstance(content, str): content = prepend + content yield id, content or "" embed_kwargs = {"store": store} if batch_size: embed_kwargs["batch_size"] = batch_size collection_obj.embed_multi(tuples(), **embed_kwargs) @cli.command() @click.argument("collection") @click.argument("id", required=False) @click.option( "-i", "--input", type=click.Path(exists=True, readable=True, allow_dash=True), help="File to embed for comparison", ) @click.option("-c", "--content", help="Content to embed for comparison") @click.option("--binary", is_flag=True, help="Treat input as binary data") @click.option( "-n", "--number", type=int, default=10, help="Number of results to return" ) @click.option("-p", "--plain", is_flag=True, help="Output in plain text format") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", ) @click.option("--prefix", help="Just IDs with this prefix", default="") def similar(collection, id, input, content, binary, number, plain, database, prefix): """ Return top N similar IDs from a collection using cosine similarity. Example usage: \b llm similar my-collection -c "I like cats" Or to find content similar to a specific stored ID: \b llm similar my-collection 1234 """ if not id and not content and not input: raise click.ClickException("Must provide content or an ID for the comparison") if database: db = sqlite_utils.Database(database) else: db = sqlite_utils.Database(user_dir() / "embeddings.db") if not db["embeddings"].exists(): raise click.ClickException("No embeddings table found in database") try: collection_obj = Collection(collection, db, create=False) except Collection.DoesNotExist: raise click.ClickException("Collection does not exist") if id: try: results = collection_obj.similar_by_id(id, number, prefix=prefix) except Collection.DoesNotExist: raise click.ClickException("ID not found in collection") else: # Resolve input text if not content: if not input or input == "-": # Read from stdin input_source = sys.stdin.buffer if binary else sys.stdin content = input_source.read() else: mode = "rb" if binary else "r" with open(input, mode) as f: content = f.read() if not content: raise click.ClickException("No content provided") results = collection_obj.similar(content, number, prefix=prefix) for result in results: if plain: click.echo(f"{result.id} ({result.score})\n") if result.content: click.echo(textwrap.indent(result.content, " ")) if result.metadata: click.echo(textwrap.indent(json.dumps(result.metadata), " ")) click.echo("") else: click.echo(json.dumps(asdict(result))) @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def embed_models(): "Manage available embedding models" @embed_models.command(name="list") @click.option( "-q", "--query", multiple=True, help="Search for embedding models matching these strings", ) def embed_models_list(query): "List available embedding models" output = [] for model_with_aliases in get_embedding_models_with_aliases(): if query: if not all(model_with_aliases.matches(q) for q in query): continue s = str(model_with_aliases.model) if model_with_aliases.aliases: s += " (aliases: {})".format(", ".join(model_with_aliases.aliases)) output.append(s) click.echo("\n".join(output)) @embed_models.command(name="default") @click.argument("model", required=False) @click.option( "--remove-default", is_flag=True, help="Reset to specifying no default model" ) def embed_models_default(model, remove_default): "Show or set the default embedding model" if not model and not remove_default: default = get_default_embedding_model() if default is None: click.echo("", err=True) else: click.echo(default) return # Validate it is a known model try: if remove_default: set_default_embedding_model(None) else: model = get_embedding_model(model) set_default_embedding_model(model.model_id) except KeyError: raise click.ClickException("Unknown embedding model: {}".format(model)) @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def collections(): "View and manage collections of embeddings" @collections.command(name="path") def collections_path(): "Output the path to the embeddings database" click.echo(user_dir() / "embeddings.db") @collections.command(name="list") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", help="Path to embeddings database", ) @click.option("json_", "--json", is_flag=True, help="Output as JSON") def embed_db_collections(database, json_): "View a list of collections" database = database or (user_dir() / "embeddings.db") db = sqlite_utils.Database(str(database)) if not db["collections"].exists(): raise click.ClickException("No collections table found in {}".format(database)) rows = db.query( """ select collections.name, collections.model, count(embeddings.id) as num_embeddings from collections left join embeddings on collections.id = embeddings.collection_id group by collections.name, collections.model """ ) if json_: click.echo(json.dumps(list(rows), indent=4)) else: for row in rows: click.echo("{}: {}".format(row["name"], row["model"])) click.echo( " {} embedding{}".format( row["num_embeddings"], "s" if row["num_embeddings"] != 1 else "" ) ) @collections.command(name="delete") @click.argument("collection") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", help="Path to embeddings database", ) def collections_delete(collection, database): """ Delete the specified collection Example usage: \b llm collections delete my-collection """ database = database or (user_dir() / "embeddings.db") db = sqlite_utils.Database(str(database)) try: collection_obj = Collection(collection, db, create=False) except Collection.DoesNotExist: raise click.ClickException("Collection does not exist") collection_obj.delete() @models.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def options(): "Manage default options for models" @options.command(name="list") def options_list(): """ List default options for all models Example usage: \b llm models options list """ options = get_all_model_options() if not options: click.echo("No default options set for any models.", err=True) return for model_id, model_options in options.items(): click.echo(f"{model_id}:") for key, value in model_options.items(): click.echo(f" {key}: {value}") @options.command(name="show") @click.argument("model") def options_show(model): """ List default options set for a specific model Example usage: \b llm models options show gpt-4o """ import llm try: # Resolve alias to model ID model_obj = llm.get_model(model) model_id = model_obj.model_id except llm.UnknownModelError: # Use as-is if not found model_id = model options = get_model_options(model_id) if not options: click.echo(f"No default options set for model '{model_id}'.", err=True) return for key, value in options.items(): click.echo(f"{key}: {value}") @options.command(name="set") @click.argument("model") @click.argument("key") @click.argument("value") def options_set(model, key, value): """ Set a default option for a model Example usage: \b llm models options set gpt-4o temperature 0.5 """ import llm try: # Resolve alias to model ID model_obj = llm.get_model(model) model_id = model_obj.model_id # Validate option against model schema try: # Create a test Options object to validate test_options = {key: value} model_obj.Options(**test_options) except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) except llm.UnknownModelError: # Use as-is if not found model_id = model set_model_option(model_id, key, value) click.echo(f"Set default option {key}={value} for model {model_id}", err=True) @options.command(name="clear") @click.argument("model") @click.argument("key", required=False) def options_clear(model, key): """ Clear default option(s) for a model Example usage: \b llm models options clear gpt-4o # Or for a single option llm models options clear gpt-4o temperature """ import llm try: # Resolve alias to model ID model_obj = llm.get_model(model) model_id = model_obj.model_id except llm.UnknownModelError: # Use as-is if not found model_id = model cleared_keys = [] if not key: cleared_keys = list(get_model_options(model_id).keys()) for key_ in cleared_keys: clear_model_option(model_id, key_) else: cleared_keys.append(key) clear_model_option(model_id, key) if cleared_keys: if len(cleared_keys) == 1: click.echo(f"Cleared option '{cleared_keys[0]}' for model {model_id}") else: click.echo( f"Cleared {', '.join(cleared_keys)} options for model {model_id}" ) def template_dir(): path = user_dir() / "templates" path.mkdir(parents=True, exist_ok=True) return path def logs_db_path(): return user_dir() / "logs.db" def get_history(chat_id): if chat_id is None: return None, [] log_path = logs_db_path() db = sqlite_utils.Database(log_path) migrate(db) if chat_id == -1: # Return the most recent chat last_row = list(db["logs"].rows_where(order_by="-id", limit=1)) if last_row: chat_id = last_row[0].get("chat_id") or last_row[0].get("id") else: # Database is empty return None, [] rows = db["logs"].rows_where( "id = ? or chat_id = ?", [chat_id, chat_id], order_by="id" ) return chat_id, rows def render_errors(errors): output = [] for error in errors: output.append(", ".join(error["loc"])) output.append(" " + error["msg"]) return "\n".join(output) load_plugins() pm.hook.register_commands(cli=cli) def _human_readable_size(size_bytes): if size_bytes == 0: return "0B" size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") i = 0 while size_bytes >= 1024 and i < len(size_name) - 1: size_bytes /= 1024.0 i += 1 return "{:.2f}{}".format(size_bytes, size_name[i]) def logs_on(): return not (user_dir() / "logs-off").exists() def get_all_model_options() -> dict: """ Get all default options for all models """ path = user_dir() / "model_options.json" if not path.exists(): return {} try: options = json.loads(path.read_text()) except json.JSONDecodeError: return {} return options def get_model_options(model_id: str) -> dict: """ Get default options for a specific model Args: model_id: Return options for model with this ID Returns: A dictionary of model options """ path = user_dir() / "model_options.json" if not path.exists(): return {} try: options = json.loads(path.read_text()) except json.JSONDecodeError: return {} return options.get(model_id, {}) def set_model_option(model_id: str, key: str, value: Any) -> None: """ Set a default option for a model. Args: model_id: The model ID key: The option key value: The option value """ path = user_dir() / "model_options.json" if path.exists(): try: options = json.loads(path.read_text()) except json.JSONDecodeError: options = {} else: options = {} # Ensure the model has an entry if model_id not in options: options[model_id] = {} # Set the option options[model_id][key] = value # Save the options path.write_text(json.dumps(options, indent=2)) def clear_model_option(model_id: str, key: str) -> None: """ Clear a model option Args: model_id: The model ID key: Key to clear """ path = user_dir() / "model_options.json" if not path.exists(): return try: options = json.loads(path.read_text()) except json.JSONDecodeError: return if model_id not in options: return if key in options[model_id]: del options[model_id][key] if not options[model_id]: del options[model_id] path.write_text(json.dumps(options, indent=2)) class LoadTemplateError(ValueError): pass def _parse_yaml_template(name, content): try: loaded = yaml.safe_load(content) except yaml.YAMLError as ex: raise LoadTemplateError("Invalid YAML: {}".format(str(ex))) if isinstance(loaded, str): return Template(name=name, prompt=loaded) loaded["name"] = name try: return Template(**loaded) except pydantic.ValidationError as ex: msg = "A validation error occurred:\n" msg += render_errors(ex.errors()) raise LoadTemplateError(msg) def load_template(name: str) -> Template: "Load template, or raise LoadTemplateError(msg)" if name.startswith("https://") or name.startswith("http://"): response = httpx.get(name) try: response.raise_for_status() except httpx.HTTPStatusError as ex: raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) return _parse_yaml_template(name, response.text) potential_path = pathlib.Path(name) if has_plugin_prefix(name) and not potential_path.exists(): prefix, rest = name.split(":", 1) loaders = get_template_loaders() if prefix not in loaders: raise LoadTemplateError("Unknown template prefix: {}".format(prefix)) loader = loaders[prefix] try: return loader(rest) except Exception as ex: raise LoadTemplateError("Could not load template {}: {}".format(name, ex)) # Try local file if potential_path.exists(): path = potential_path else: # Look for template in template_dir() path = template_dir() / f"{name}.yaml" if not path.exists(): raise LoadTemplateError(f"Invalid template: {name}") content = path.read_text() template_obj = _parse_yaml_template(name, content) # We trust functions here because they came from the filesystem template_obj._functions_is_trusted = True return template_obj def _tools_from_code(code_or_path: str) -> List[Tool]: """ Treat all Python functions in the code as tools """ if "\n" not in code_or_path and code_or_path.endswith(".py"): try: code_or_path = pathlib.Path(code_or_path).read_text() except FileNotFoundError: raise click.ClickException("File not found: {}".format(code_or_path)) namespace: Dict[str, Any] = {} tools = [] try: exec(code_or_path, namespace) except SyntaxError as ex: raise click.ClickException("Error in --functions definition: {}".format(ex)) # Register all callables in the locals dict: for name, value in namespace.items(): if callable(value) and not name.startswith("_"): tools.append(Tool.function(value)) return tools def _debug_tool_call(_, tool_call, tool_result): click.echo( click.style( "\nTool call: {}({})".format(tool_call.name, tool_call.arguments), fg="yellow", bold=True, ), err=True, ) output = "" attachments = "" if tool_result.attachments: attachments += "\nAttachments:\n" for attachment in tool_result.attachments: attachments += f" {repr(attachment)}\n" try: output = json.dumps(json.loads(tool_result.output), indent=2) except ValueError: output = tool_result.output output += attachments click.echo( click.style( textwrap.indent(output, " ") + ("\n" if not tool_result.exception else ""), fg="green", bold=True, ), err=True, ) if tool_result.exception: click.echo( click.style( " Exception: {}".format(tool_result.exception), fg="red", bold=True, ), err=True, ) def _approve_tool_call(_, tool_call): click.echo( click.style( "Tool call: {}({})".format(tool_call.name, tool_call.arguments), fg="yellow", bold=True, ), err=True, ) if not click.confirm("Approve tool call?"): raise CancelToolCall("User cancelled tool call") def _gather_tools( tool_specs: List[str], python_tools: List[str] ) -> List[Union[Tool, Type[Toolbox]]]: tools: List[Union[Tool, Type[Toolbox]]] = [] if python_tools: for code_or_path in python_tools: tools.extend(_tools_from_code(code_or_path)) registered_tools = get_tools() registered_classes = dict( (key, value) for key, value in registered_tools.items() if inspect.isclass(value) ) bad_tools = [ tool for tool in tool_specs if tool.split("(")[0] not in registered_tools ] if bad_tools: raise click.ClickException( "Tool(s) {} not found. Available tools: {}".format( ", ".join(bad_tools), ", ".join(registered_tools.keys()) ) ) for tool_spec in tool_specs: if not tool_spec[0].isupper(): # It's a function tools.append(registered_tools[tool_spec]) else: # It's a class tools.append(instantiate_from_spec(registered_classes, tool_spec)) return tools def _get_conversation_tools(conversation, tools): if conversation and not tools and conversation.responses: # Copy plugin tools from first response in conversation initial_tools = conversation.responses[0].prompt.tools if initial_tools: # Only tools from plugins: return [tool.name for tool in initial_tools if tool.plugin] ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1754975852.910846 llm-0.27.1/llm/default_plugins/0000755000175100001660000000000015046547155015755 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/default_plugins/__init__.py0000644000175100001660000000000015046547145020053 0ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/default_plugins/default_tools.py0000644000175100001660000000022615046547145021172 0ustar00runnerdockerimport llm from llm.tools import llm_time, llm_version @llm.hookimpl def register_tools(register): register(llm_version) register(llm_time) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/default_plugins/openai_models.py0000644000175100001660000010467715046547145021163 0ustar00runnerdockerfrom llm import AsyncKeyModel, EmbeddingModel, KeyModel, hookimpl import llm from llm.utils import ( dicts_to_table_string, remove_dict_none_values, logging_client, simplify_usage_dict, ) import click import datetime from enum import Enum import httpx import openai import os from pydantic import field_validator, Field from typing import AsyncGenerator, List, Iterable, Iterator, Optional, Union import json import yaml @hookimpl def register_models(register): # GPT-4o register( Chat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), AsyncChat("gpt-4o", vision=True, supports_schema=True, supports_tools=True), aliases=("4o",), ) register( Chat("chatgpt-4o-latest", vision=True), AsyncChat("chatgpt-4o-latest", vision=True), aliases=("chatgpt-4o",), ) register( Chat("gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True), AsyncChat( "gpt-4o-mini", vision=True, supports_schema=True, supports_tools=True ), aliases=("4o-mini",), ) for audio_model_id in ( "gpt-4o-audio-preview", "gpt-4o-audio-preview-2024-12-17", "gpt-4o-audio-preview-2024-10-01", "gpt-4o-mini-audio-preview", "gpt-4o-mini-audio-preview-2024-12-17", ): register( Chat(audio_model_id, audio=True), AsyncChat(audio_model_id, audio=True), ) # GPT-4.1 for model_id in ("gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano"): register( Chat(model_id, vision=True, supports_schema=True, supports_tools=True), AsyncChat(model_id, vision=True, supports_schema=True, supports_tools=True), aliases=(model_id.replace("gpt-", ""),), ) # 3.5 and 4 register( Chat("gpt-3.5-turbo"), AsyncChat("gpt-3.5-turbo"), aliases=("3.5", "chatgpt") ) register( Chat("gpt-3.5-turbo-16k"), AsyncChat("gpt-3.5-turbo-16k"), aliases=("chatgpt-16k", "3.5-16k"), ) register(Chat("gpt-4"), AsyncChat("gpt-4"), aliases=("4", "gpt4")) register(Chat("gpt-4-32k"), AsyncChat("gpt-4-32k"), aliases=("4-32k",)) # GPT-4 Turbo models register(Chat("gpt-4-1106-preview"), AsyncChat("gpt-4-1106-preview")) register(Chat("gpt-4-0125-preview"), AsyncChat("gpt-4-0125-preview")) register(Chat("gpt-4-turbo-2024-04-09"), AsyncChat("gpt-4-turbo-2024-04-09")) register( Chat("gpt-4-turbo"), AsyncChat("gpt-4-turbo"), aliases=("gpt-4-turbo-preview", "4-turbo", "4t"), ) # GPT-4.5 register( Chat( "gpt-4.5-preview-2025-02-27", vision=True, supports_schema=True, supports_tools=True, ), AsyncChat( "gpt-4.5-preview-2025-02-27", vision=True, supports_schema=True, supports_tools=True, ), ) register( Chat("gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True), AsyncChat( "gpt-4.5-preview", vision=True, supports_schema=True, supports_tools=True ), aliases=("gpt-4.5",), ) # o1 for model_id in ("o1", "o1-2024-12-17"): register( Chat( model_id, vision=True, can_stream=False, reasoning=True, supports_schema=True, supports_tools=True, ), AsyncChat( model_id, vision=True, can_stream=False, reasoning=True, supports_schema=True, supports_tools=True, ), ) register( Chat("o1-preview", allows_system_prompt=False), AsyncChat("o1-preview", allows_system_prompt=False), ) register( Chat("o1-mini", allows_system_prompt=False), AsyncChat("o1-mini", allows_system_prompt=False), ) register( Chat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), AsyncChat("o3-mini", reasoning=True, supports_schema=True, supports_tools=True), ) register( Chat( "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True ), AsyncChat( "o3", vision=True, reasoning=True, supports_schema=True, supports_tools=True ), ) register( Chat( "o4-mini", vision=True, reasoning=True, supports_schema=True, supports_tools=True, ), AsyncChat( "o4-mini", vision=True, reasoning=True, supports_schema=True, supports_tools=True, ), ) # GPT-5 for model_id in ( "gpt-5", "gpt-5-mini", "gpt-5-nano", "gpt-5-2025-08-07", "gpt-5-mini-2025-08-07", "gpt-5-nano-2025-08-07", ): register( Chat( model_id, vision=True, reasoning=True, supports_schema=True, supports_tools=True, ), AsyncChat( model_id, vision=True, reasoning=True, supports_schema=True, supports_tools=True, ), ) # The -instruct completion model register( Completion("gpt-3.5-turbo-instruct", default_max_tokens=256), aliases=("3.5-instruct", "chatgpt-instruct"), ) # Load extra models extra_path = llm.user_dir() / "extra-openai-models.yaml" if not extra_path.exists(): return with open(extra_path) as f: extra_models = yaml.safe_load(f) for extra_model in extra_models: model_id = extra_model["model_id"] aliases = extra_model.get("aliases", []) model_name = extra_model["model_name"] api_base = extra_model.get("api_base") api_type = extra_model.get("api_type") api_version = extra_model.get("api_version") api_engine = extra_model.get("api_engine") headers = extra_model.get("headers") reasoning = extra_model.get("reasoning") kwargs = {} if extra_model.get("can_stream") is False: kwargs["can_stream"] = False if extra_model.get("supports_schema") is True: kwargs["supports_schema"] = True if extra_model.get("supports_tools") is True: kwargs["supports_tools"] = True if extra_model.get("vision") is True: kwargs["vision"] = True if extra_model.get("audio") is True: kwargs["audio"] = True if extra_model.get("completion"): klass = Completion else: klass = Chat chat_model = klass( model_id, model_name=model_name, api_base=api_base, api_type=api_type, api_version=api_version, api_engine=api_engine, headers=headers, reasoning=reasoning, **kwargs, ) if api_base: chat_model.needs_key = None if extra_model.get("api_key_name"): chat_model.needs_key = extra_model["api_key_name"] register( chat_model, aliases=aliases, ) @hookimpl def register_embedding_models(register): register( OpenAIEmbeddingModel("text-embedding-ada-002", "text-embedding-ada-002"), aliases=( "ada", "ada-002", ), ) register( OpenAIEmbeddingModel("text-embedding-3-small", "text-embedding-3-small"), aliases=("3-small",), ) register( OpenAIEmbeddingModel("text-embedding-3-large", "text-embedding-3-large"), aliases=("3-large",), ) # With varying dimensions register( OpenAIEmbeddingModel( "text-embedding-3-small-512", "text-embedding-3-small", 512 ), aliases=("3-small-512",), ) register( OpenAIEmbeddingModel( "text-embedding-3-large-256", "text-embedding-3-large", 256 ), aliases=("3-large-256",), ) register( OpenAIEmbeddingModel( "text-embedding-3-large-1024", "text-embedding-3-large", 1024 ), aliases=("3-large-1024",), ) class OpenAIEmbeddingModel(EmbeddingModel): needs_key = "openai" key_env_var = "OPENAI_API_KEY" batch_size = 100 def __init__(self, model_id, openai_model_id, dimensions=None): self.model_id = model_id self.openai_model_id = openai_model_id self.dimensions = dimensions def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: kwargs = { "input": items, "model": self.openai_model_id, } if self.dimensions: kwargs["dimensions"] = self.dimensions client = openai.OpenAI(api_key=self.get_key()) results = client.embeddings.create(**kwargs).data return ([float(r) for r in result.embedding] for result in results) @hookimpl def register_commands(cli): @cli.group(name="openai") def openai_(): "Commands for working directly with the OpenAI API" @openai_.command() @click.option("json_", "--json", is_flag=True, help="Output as JSON") @click.option("--key", help="OpenAI API key") def models(json_, key): "List models available to you from the OpenAI API" from llm import get_key api_key = get_key(key, "openai", "OPENAI_API_KEY") response = httpx.get( "https://api.openai.com/v1/models", headers={"Authorization": f"Bearer {api_key}"}, ) if response.status_code != 200: raise click.ClickException( f"Error {response.status_code} from OpenAI API: {response.text}" ) models = response.json()["data"] if json_: click.echo(json.dumps(models, indent=4)) else: to_print = [] for model in models: # Print id, owned_by, root, created as ISO 8601 created_str = datetime.datetime.fromtimestamp( model["created"], datetime.timezone.utc ).isoformat() to_print.append( { "id": model["id"], "owned_by": model["owned_by"], "created": created_str, } ) done = dicts_to_table_string("id owned_by created".split(), to_print) print("\n".join(done)) class SharedOptions(llm.Options): temperature: Optional[float] = Field( description=( "What sampling temperature to use, between 0 and 2. Higher values like " "0.8 will make the output more random, while lower values like 0.2 will " "make it more focused and deterministic." ), ge=0, le=2, default=None, ) max_tokens: Optional[int] = Field( description="Maximum number of tokens to generate.", default=None ) top_p: Optional[float] = Field( description=( "An alternative to sampling with temperature, called nucleus sampling, " "where the model considers the results of the tokens with top_p " "probability mass. So 0.1 means only the tokens comprising the top " "10% probability mass are considered. Recommended to use top_p or " "temperature but not both." ), ge=0, le=1, default=None, ) frequency_penalty: Optional[float] = Field( description=( "Number between -2.0 and 2.0. Positive values penalize new tokens based " "on their existing frequency in the text so far, decreasing the model's " "likelihood to repeat the same line verbatim." ), ge=-2, le=2, default=None, ) presence_penalty: Optional[float] = Field( description=( "Number between -2.0 and 2.0. Positive values penalize new tokens based " "on whether they appear in the text so far, increasing the model's " "likelihood to talk about new topics." ), ge=-2, le=2, default=None, ) stop: Optional[str] = Field( description=("A string where the API will stop generating further tokens."), default=None, ) logit_bias: Optional[Union[dict, str]] = Field( description=( "Modify the likelihood of specified tokens appearing in the completion. " 'Pass a JSON string like \'{"1712":-100, "892":-100, "1489":-100}\'' ), default=None, ) seed: Optional[int] = Field( description="Integer seed to attempt to sample deterministically", default=None, ) @field_validator("logit_bias") def validate_logit_bias(cls, logit_bias): if logit_bias is None: return None if isinstance(logit_bias, str): try: logit_bias = json.loads(logit_bias) except json.JSONDecodeError: raise ValueError("Invalid JSON in logit_bias string") validated_logit_bias = {} for key, value in logit_bias.items(): try: int_key = int(key) int_value = int(value) if -100 <= int_value <= 100: validated_logit_bias[int_key] = int_value else: raise ValueError("Value must be between -100 and 100") except ValueError: raise ValueError("Invalid key-value pair in logit_bias dictionary") return validated_logit_bias class ReasoningEffortEnum(str, Enum): minimal = "minimal" low = "low" medium = "medium" high = "high" class OptionsForReasoning(SharedOptions): json_object: Optional[bool] = Field( description="Output a valid JSON object {...}. Prompt must mention JSON.", default=None, ) reasoning_effort: Optional[ReasoningEffortEnum] = Field( description=( "Constraints effort on reasoning for reasoning models. Currently supported " "values are low, medium, and high. Reducing reasoning effort can result in " "faster responses and fewer tokens used on reasoning in a response." ), default=None, ) def _attachment(attachment): url = attachment.url base64_content = "" if not url or attachment.resolve_type().startswith("audio/"): base64_content = attachment.base64_content() url = f"data:{attachment.resolve_type()};base64,{base64_content}" if attachment.resolve_type() == "application/pdf": if not base64_content: base64_content = attachment.base64_content() return { "type": "file", "file": { "filename": f"{attachment.id()}.pdf", "file_data": f"data:application/pdf;base64,{base64_content}", }, } if attachment.resolve_type().startswith("image/"): return {"type": "image_url", "image_url": {"url": url}} else: format_ = "wav" if attachment.resolve_type() == "audio/wav" else "mp3" return { "type": "input_audio", "input_audio": { "data": base64_content, "format": format_, }, } class _Shared: def __init__( self, model_id, key=None, model_name=None, api_base=None, api_type=None, api_version=None, api_engine=None, headers=None, can_stream=True, vision=False, audio=False, reasoning=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, ): self.model_id = model_id self.key = key self.supports_schema = supports_schema self.supports_tools = supports_tools self.model_name = model_name self.api_base = api_base self.api_type = api_type self.api_version = api_version self.api_engine = api_engine self.headers = headers self.can_stream = can_stream self.vision = vision self.allows_system_prompt = allows_system_prompt self.attachment_types = set() if reasoning: self.Options = OptionsForReasoning if vision: self.attachment_types.update( { "image/png", "image/jpeg", "image/webp", "image/gif", "application/pdf", } ) if audio: self.attachment_types.update( { "audio/wav", "audio/mpeg", } ) def __str__(self): return "OpenAI Chat: {}".format(self.model_id) def build_messages(self, prompt, conversation): messages = [] current_system = None if conversation is not None: for prev_response in conversation.responses: if ( prev_response.prompt.system and prev_response.prompt.system != current_system ): messages.append( {"role": "system", "content": prev_response.prompt.system} ) current_system = prev_response.prompt.system if prev_response.attachments: attachment_message = [] if prev_response.prompt.prompt: attachment_message.append( {"type": "text", "text": prev_response.prompt.prompt} ) for attachment in prev_response.attachments: attachment_message.append(_attachment(attachment)) messages.append({"role": "user", "content": attachment_message}) elif prev_response.prompt.prompt: messages.append( {"role": "user", "content": prev_response.prompt.prompt} ) for tool_result in prev_response.prompt.tool_results: messages.append( { "role": "tool", "tool_call_id": tool_result.tool_call_id, "content": tool_result.output, } ) prev_text = prev_response.text_or_raise() if prev_text: messages.append({"role": "assistant", "content": prev_text}) tool_calls = prev_response.tool_calls_or_raise() if tool_calls: messages.append( { "role": "assistant", "tool_calls": [ { "type": "function", "id": tool_call.tool_call_id, "function": { "name": tool_call.name, "arguments": json.dumps(tool_call.arguments), }, } for tool_call in tool_calls ], } ) if prompt.system and prompt.system != current_system: messages.append({"role": "system", "content": prompt.system}) for tool_result in prompt.tool_results: messages.append( { "role": "tool", "tool_call_id": tool_result.tool_call_id, "content": tool_result.output, } ) if not prompt.attachments: if prompt.prompt: messages.append({"role": "user", "content": prompt.prompt or ""}) else: attachment_message = [] if prompt.prompt: attachment_message.append({"type": "text", "text": prompt.prompt}) for attachment in prompt.attachments: attachment_message.append(_attachment(attachment)) messages.append({"role": "user", "content": attachment_message}) return messages def set_usage(self, response, usage): if not usage: return input_tokens = usage.pop("prompt_tokens") output_tokens = usage.pop("completion_tokens") usage.pop("total_tokens") response.set_usage( input=input_tokens, output=output_tokens, details=simplify_usage_dict(usage) ) def get_client(self, key, *, async_=False): kwargs = {} if self.api_base: kwargs["base_url"] = self.api_base if self.api_type: kwargs["api_type"] = self.api_type if self.api_version: kwargs["api_version"] = self.api_version if self.api_engine: kwargs["engine"] = self.api_engine if self.needs_key: kwargs["api_key"] = self.get_key(key) else: # OpenAI-compatible models don't need a key, but the # openai client library requires one kwargs["api_key"] = "DUMMY_KEY" if self.headers: kwargs["default_headers"] = self.headers if os.environ.get("LLM_OPENAI_SHOW_RESPONSES"): kwargs["http_client"] = logging_client() if async_: return openai.AsyncOpenAI(**kwargs) else: return openai.OpenAI(**kwargs) def build_kwargs(self, prompt, stream): kwargs = dict(not_nulls(prompt.options)) json_object = kwargs.pop("json_object", None) if "max_tokens" not in kwargs and self.default_max_tokens is not None: kwargs["max_tokens"] = self.default_max_tokens if json_object: kwargs["response_format"] = {"type": "json_object"} if prompt.schema: kwargs["response_format"] = { "type": "json_schema", "json_schema": {"name": "output", "schema": prompt.schema}, } if prompt.tools: kwargs["tools"] = [ { "type": "function", "function": { "name": tool.name, "description": tool.description or None, "parameters": tool.input_schema, }, } for tool in prompt.tools ] if stream: kwargs["stream_options"] = {"include_usage": True} return kwargs class Chat(_Shared, KeyModel): needs_key = "openai" key_env_var = "OPENAI_API_KEY" default_max_tokens = None class Options(SharedOptions): json_object: Optional[bool] = Field( description="Output a valid JSON object {...}. Prompt must mention JSON.", default=None, ) def execute(self, prompt, stream, response, conversation=None, key=None): if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") messages = self.build_messages(prompt, conversation) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key) usage = None if stream: completion = client.chat.completions.create( model=self.model_name or self.model_id, messages=messages, stream=True, **kwargs, ) chunks = [] tool_calls = {} for chunk in completion: chunks.append(chunk) if chunk.usage: usage = chunk.usage.model_dump() if chunk.choices and chunk.choices[0].delta: for tool_call in chunk.choices[0].delta.tool_calls or []: if tool_call.function.arguments is None: tool_call.function.arguments = "" index = tool_call.index if index not in tool_calls: tool_calls[index] = tool_call else: tool_calls[ index ].function.arguments += tool_call.function.arguments try: content = chunk.choices[0].delta.content except IndexError: content = None if content is not None: yield content response.response_json = remove_dict_none_values(combine_chunks(chunks)) if tool_calls: for value in tool_calls.values(): # value.function looks like this: # ChoiceDeltaToolCallFunction(arguments='{"city":"San Francisco"}', name='get_weather') response.add_tool_call( llm.ToolCall( tool_call_id=value.id, name=value.function.name, arguments=json.loads(value.function.arguments), ) ) else: completion = client.chat.completions.create( model=self.model_name or self.model_id, messages=messages, stream=False, **kwargs, ) usage = completion.usage.model_dump() response.response_json = remove_dict_none_values(completion.model_dump()) for tool_call in completion.choices[0].message.tool_calls or []: response.add_tool_call( llm.ToolCall( tool_call_id=tool_call.id, name=tool_call.function.name, arguments=json.loads(tool_call.function.arguments), ) ) if completion.choices[0].message.content is not None: yield completion.choices[0].message.content self.set_usage(response, usage) response._prompt_json = redact_data({"messages": messages}) class AsyncChat(_Shared, AsyncKeyModel): needs_key = "openai" key_env_var = "OPENAI_API_KEY" default_max_tokens = None class Options(SharedOptions): json_object: Optional[bool] = Field( description="Output a valid JSON object {...}. Prompt must mention JSON.", default=None, ) async def execute( self, prompt, stream, response, conversation=None, key=None ) -> AsyncGenerator[str, None]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") messages = self.build_messages(prompt, conversation) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key, async_=True) usage = None if stream: completion = await client.chat.completions.create( model=self.model_name or self.model_id, messages=messages, stream=True, **kwargs, ) chunks = [] tool_calls = {} async for chunk in completion: if chunk.usage: usage = chunk.usage.model_dump() chunks.append(chunk) if chunk.usage: usage = chunk.usage.model_dump() if chunk.choices and chunk.choices[0].delta: for tool_call in chunk.choices[0].delta.tool_calls or []: if tool_call.function.arguments is None: tool_call.function.arguments = "" index = tool_call.index if index not in tool_calls: tool_calls[index] = tool_call else: tool_calls[ index ].function.arguments += tool_call.function.arguments try: content = chunk.choices[0].delta.content except IndexError: content = None if content is not None: yield content if tool_calls: for value in tool_calls.values(): # value.function looks like this: # ChoiceDeltaToolCallFunction(arguments='{"city":"San Francisco"}', name='get_weather') response.add_tool_call( llm.ToolCall( tool_call_id=value.id, name=value.function.name, arguments=json.loads(value.function.arguments), ) ) response.response_json = remove_dict_none_values(combine_chunks(chunks)) else: completion = await client.chat.completions.create( model=self.model_name or self.model_id, messages=messages, stream=False, **kwargs, ) response.response_json = remove_dict_none_values(completion.model_dump()) usage = completion.usage.model_dump() for tool_call in completion.choices[0].message.tool_calls or []: response.add_tool_call( llm.ToolCall( tool_call_id=tool_call.id, name=tool_call.function.name, arguments=json.loads(tool_call.function.arguments), ) ) if completion.choices[0].message.content is not None: yield completion.choices[0].message.content self.set_usage(response, usage) response._prompt_json = redact_data({"messages": messages}) class Completion(Chat): class Options(SharedOptions): logprobs: Optional[int] = Field( description="Include the log probabilities of most likely N per token", default=None, le=5, ) def __init__(self, *args, default_max_tokens=None, **kwargs): super().__init__(*args, **kwargs) self.default_max_tokens = default_max_tokens def __str__(self): return "OpenAI Completion: {}".format(self.model_id) def execute(self, prompt, stream, response, conversation=None, key=None): if prompt.system: raise NotImplementedError( "System prompts are not supported for OpenAI completion models" ) messages = [] if conversation is not None: for prev_response in conversation.responses: messages.append(prev_response.prompt.prompt) messages.append(prev_response.text()) messages.append(prompt.prompt) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key) if stream: completion = client.completions.create( model=self.model_name or self.model_id, prompt="\n".join(messages), stream=True, **kwargs, ) chunks = [] for chunk in completion: chunks.append(chunk) try: content = chunk.choices[0].text except IndexError: content = None if content is not None: yield content combined = combine_chunks(chunks) cleaned = remove_dict_none_values(combined) response.response_json = cleaned else: completion = client.completions.create( model=self.model_name or self.model_id, prompt="\n".join(messages), stream=False, **kwargs, ) response.response_json = remove_dict_none_values(completion.model_dump()) yield completion.choices[0].text response._prompt_json = redact_data({"messages": messages}) def not_nulls(data) -> dict: return {key: value for key, value in data if value is not None} def combine_chunks(chunks: List) -> dict: content = "" role = None finish_reason = None # If any of them have log probability, we're going to persist # those later on logprobs = [] usage = {} for item in chunks: if item.usage: usage = item.usage.model_dump() for choice in item.choices: if choice.logprobs and hasattr(choice.logprobs, "top_logprobs"): logprobs.append( { "text": choice.text if hasattr(choice, "text") else None, "top_logprobs": choice.logprobs.top_logprobs, } ) if not hasattr(choice, "delta"): content += choice.text continue role = choice.delta.role if choice.delta.content is not None: content += choice.delta.content if choice.finish_reason is not None: finish_reason = choice.finish_reason # Imitations of the OpenAI API may be missing some of these fields combined = { "content": content, "role": role, "finish_reason": finish_reason, "usage": usage, } if logprobs: combined["logprobs"] = logprobs if chunks: for key in ("id", "object", "model", "created", "index"): value = getattr(chunks[0], key, None) if value is not None: combined[key] = value return combined def redact_data(input_dict): """ Recursively search through the input dictionary for any 'image_url' keys and modify the 'url' value to be just 'data:...'. Also redact input_audio.data keys """ if isinstance(input_dict, dict): for key, value in input_dict.items(): if ( key == "image_url" and isinstance(value, dict) and "url" in value and value["url"].startswith("data:") ): value["url"] = "data:..." elif key == "input_audio" and isinstance(value, dict) and "data" in value: value["data"] = "..." else: redact_data(value) elif isinstance(input_dict, list): for item in input_dict: redact_data(item) return input_dict ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/embeddings.py0000644000175100001660000003070615046547145015250 0ustar00runnerdockerfrom .models import EmbeddingModel from .embeddings_migrations import embeddings_migrations from dataclasses import dataclass import hashlib from itertools import islice import json from sqlite_utils import Database from sqlite_utils.db import Table import time from typing import cast, Any, Dict, Iterable, List, Optional, Tuple, Union @dataclass class Entry: id: str score: Optional[float] content: Optional[str] = None metadata: Optional[Dict[str, Any]] = None class Collection: class DoesNotExist(Exception): pass def __init__( self, name: str, db: Optional[Database] = None, *, model: Optional[EmbeddingModel] = None, model_id: Optional[str] = None, create: bool = True, ) -> None: """ A collection of embeddings Returns the collection with the given name, creating it if it does not exist. If you set create=False a Collection.DoesNotExist exception will be raised if the collection does not already exist. Args: db (sqlite_utils.Database): Database to store the collection in name (str): Name of the collection model (llm.models.EmbeddingModel, optional): Embedding model to use model_id (str, optional): Alternatively, ID of the embedding model to use create (bool, optional): Whether to create the collection if it does not exist """ import llm self.db = db or Database(memory=True) self.name = name self._model = model embeddings_migrations.apply(self.db) rows = list(self.db["collections"].rows_where("name = ?", [self.name])) if rows: row = rows[0] self.id = row["id"] self.model_id = row["model"] else: if create: # Collection does not exist, so model or model_id is required if not model and not model_id: raise ValueError( "Either model= or model_id= must be provided when creating a new collection" ) # Create it if model_id: # Resolve alias model = llm.get_embedding_model(model_id) self._model = model model_id = cast(EmbeddingModel, model).model_id self.id = ( cast(Table, self.db["collections"]) .insert( { "name": self.name, "model": model_id, } ) .last_pk ) else: raise self.DoesNotExist(f"Collection '{name}' does not exist") def model(self) -> EmbeddingModel: "Return the embedding model used by this collection" import llm if self._model is None: self._model = llm.get_embedding_model(self.model_id) return cast(EmbeddingModel, self._model) def count(self) -> int: """ Count the number of items in the collection. Returns: int: Number of items in the collection """ return next( self.db.query( """ select count(*) as c from embeddings where collection_id = ( select id from collections where name = ? ) """, (self.name,), ) )["c"] def embed( self, id: str, value: Union[str, bytes], metadata: Optional[Dict[str, Any]] = None, store: bool = False, ) -> None: """ Embed value and store it in the collection with a given ID. Args: id (str): ID for the value value (str or bytes): value to be embedded metadata (dict, optional): Metadata to be stored store (bool, optional): Whether to store the value in the content or content_blob column """ from llm import encode content_hash = self.content_hash(value) if self.db["embeddings"].count_where( "content_hash = ? and collection_id = ?", [content_hash, self.id] ): return embedding = self.model().embed(value) cast(Table, self.db["embeddings"]).insert( { "collection_id": self.id, "id": id, "embedding": encode(embedding), "content": value if (store and isinstance(value, str)) else None, "content_blob": value if (store and isinstance(value, bytes)) else None, "content_hash": content_hash, "metadata": json.dumps(metadata) if metadata else None, "updated": int(time.time()), }, replace=True, ) def embed_multi( self, entries: Iterable[Tuple[str, Union[str, bytes]]], store: bool = False, batch_size: int = 100, ) -> None: """ Embed multiple texts and store them in the collection with given IDs. Args: entries (iterable): Iterable of (id: str, text: str) tuples store (bool, optional): Whether to store the text in the content column batch_size (int, optional): custom maximum batch size to use """ self.embed_multi_with_metadata( ((id, value, None) for id, value in entries), store=store, batch_size=batch_size, ) def embed_multi_with_metadata( self, entries: Iterable[Tuple[str, Union[str, bytes], Optional[Dict[str, Any]]]], store: bool = False, batch_size: int = 100, ) -> None: """ Embed multiple values along with metadata and store them in the collection with given IDs. Args: entries (iterable): Iterable of (id: str, value: str or bytes, metadata: None or dict) store (bool, optional): Whether to store the value in the content or content_blob column batch_size (int, optional): custom maximum batch size to use """ import llm batch_size = min(batch_size, (self.model().batch_size or batch_size)) iterator = iter(entries) collection_id = self.id while True: batch = list(islice(iterator, batch_size)) if not batch: break # Calculate hashes first items_and_hashes = [(item, self.content_hash(item[1])) for item in batch] # Any of those hashes already exist? existing_ids = [ row["id"] for row in self.db.query( """ select id from embeddings where collection_id = ? and content_hash in ({}) """.format( ",".join("?" for _ in items_and_hashes) ), [collection_id] + [item_and_hash[1] for item_and_hash in items_and_hashes], ) ] filtered_batch = [item for item in batch if item[0] not in existing_ids] embeddings = list( self.model().embed_multi(item[1] for item in filtered_batch) ) with self.db.conn: cast(Table, self.db["embeddings"]).insert_all( ( { "collection_id": collection_id, "id": id, "embedding": llm.encode(embedding), "content": ( value if (store and isinstance(value, str)) else None ), "content_blob": ( value if (store and isinstance(value, bytes)) else None ), "content_hash": self.content_hash(value), "metadata": json.dumps(metadata) if metadata else None, "updated": int(time.time()), } for (embedding, (id, value, metadata)) in zip( embeddings, filtered_batch ) ), replace=True, ) def similar_by_vector( self, vector: List[float], number: int = 10, skip_id: Optional[str] = None, prefix: Optional[str] = None, ) -> List[Entry]: """ Find similar items in the collection by a given vector. Args: vector (list): Vector to search by number (int, optional): Number of similar items to return skip_id (str, optional): An ID to exclude from the results prefix: (str, optional): Filter results to IDs witih this prefix Returns: list: List of Entry objects """ import llm def distance_score(other_encoded): other_vector = llm.decode(other_encoded) return llm.cosine_similarity(other_vector, vector) self.db.register_function(distance_score, replace=True) where_bits = ["collection_id = ?"] where_args = [str(self.id)] if prefix: where_bits.append("id LIKE ? || '%'") where_args.append(prefix) if skip_id: where_bits.append("id != ?") where_args.append(skip_id) return [ Entry( id=row["id"], score=row["score"], content=row["content"], metadata=json.loads(row["metadata"]) if row["metadata"] else None, ) for row in self.db.query( """ select id, content, metadata, distance_score(embedding) as score from embeddings where {where} order by score desc limit {number} """.format( where=" and ".join(where_bits), number=number, ), where_args, ) ] def similar_by_id( self, id: str, number: int = 10, prefix: Optional[str] = None ) -> List[Entry]: """ Find similar items in the collection by a given ID. Args: id (str): ID to search by number (int, optional): Number of similar items to return prefix: (str, optional): Filter results to IDs with this prefix Returns: list: List of Entry objects """ import llm matches = list( self.db["embeddings"].rows_where( "collection_id = ? and id = ?", (self.id, id) ) ) if not matches: raise self.DoesNotExist("ID not found") embedding = matches[0]["embedding"] comparison_vector = llm.decode(embedding) return self.similar_by_vector( comparison_vector, number, skip_id=id, prefix=prefix ) def similar( self, value: Union[str, bytes], number: int = 10, prefix: Optional[str] = None ) -> List[Entry]: """ Find similar items in the collection by a given value. Args: value (str or bytes): value to search by number (int, optional): Number of similar items to return prefix: (str, optional): Filter results to IDs with this prefix Returns: list: List of Entry objects """ comparison_vector = self.model().embed(value) return self.similar_by_vector(comparison_vector, number, prefix=prefix) @classmethod def exists(cls, db: Database, name: str) -> bool: """ Does this collection exist in the database? Args: name (str): Name of the collection """ rows = list(db["collections"].rows_where("name = ?", [name])) return bool(rows) def delete(self): """ Delete the collection and its embeddings from the database """ with self.db.conn: self.db.execute("delete from embeddings where collection_id = ?", [self.id]) self.db.execute("delete from collections where id = ?", [self.id]) @staticmethod def content_hash(input: Union[str, bytes]) -> bytes: "Hash content for deduplication. Override to change hashing behavior." if isinstance(input, str): input = input.encode("utf8") return hashlib.md5(input).digest() ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/embeddings_migrations.py0000644000175100001660000000467215046547145017507 0ustar00runnerdockerfrom sqlite_migrate import Migrations import hashlib import time embeddings_migrations = Migrations("llm.embeddings") @embeddings_migrations() def m001_create_tables(db): db["collections"].create({"id": int, "name": str, "model": str}, pk="id") db["collections"].create_index(["name"], unique=True) db["embeddings"].create( { "collection_id": int, "id": str, "embedding": bytes, "content": str, "metadata": str, }, pk=("collection_id", "id"), ) @embeddings_migrations() def m002_foreign_key(db): db["embeddings"].add_foreign_key("collection_id", "collections", "id") @embeddings_migrations() def m003_add_updated(db): db["embeddings"].add_column("updated", int) # Pretty-print the schema db["embeddings"].transform() # Assume anything existing was last updated right now db.query( "update embeddings set updated = ? where updated is null", [int(time.time())] ) @embeddings_migrations() def m004_store_content_hash(db): db["embeddings"].add_column("content_hash", bytes) db["embeddings"].transform( column_order=( "collection_id", "id", "embedding", "content", "content_hash", "metadata", "updated", ) ) # Register functions manually so we can de-register later def md5(text): return hashlib.md5(text.encode("utf8")).digest() def random_md5(): return hashlib.md5(str(time.time()).encode("utf8")).digest() db.conn.create_function("temp_md5", 1, md5) db.conn.create_function("temp_random_md5", 0, random_md5) with db.conn: db.execute( """ update embeddings set content_hash = temp_md5(content) where content is not null """ ) db.execute( """ update embeddings set content_hash = temp_random_md5() where content is null """ ) db["embeddings"].create_index(["content_hash"]) # De-register functions db.conn.create_function("temp_md5", 1, None) db.conn.create_function("temp_random_md5", 0, None) @embeddings_migrations() def m005_add_content_blob(db): db["embeddings"].add_column("content_blob", bytes) db["embeddings"].transform( column_order=("collection_id", "id", "embedding", "content", "content_blob") ) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/errors.py0000644000175100001660000000030415046547145014452 0ustar00runnerdockerclass ModelError(Exception): "Models can raise this error, which will be displayed to the user" class NeedsKeyException(ModelError): "Model needs an API key which has not been provided" ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/hookspecs.py0000644000175100001660000000146715046547145015147 0ustar00runnerdockerfrom pluggy import HookimplMarker from pluggy import HookspecMarker hookspec = HookspecMarker("llm") hookimpl = HookimplMarker("llm") @hookspec def register_commands(cli): """Register additional CLI commands, e.g. 'llm mycommand ...'""" @hookspec def register_models(register): "Register additional model instances representing LLM models that can be called" @hookspec def register_embedding_models(register): "Register additional model instances that can be used for embedding" @hookspec def register_template_loaders(register): "Register additional template loaders with prefixes" @hookspec def register_fragment_loaders(register): "Register additional fragment loaders with prefixes" @hookspec def register_tools(register): "Register functions that can be used as tools by the LLMs" ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/migrations.py0000644000175100001660000002421215046547145015316 0ustar00runnerdockerimport datetime from typing import Callable, List MIGRATIONS: List[Callable] = [] migration = MIGRATIONS.append def migrate(db): ensure_migrations_table(db) already_applied = {r["name"] for r in db["_llm_migrations"].rows} for fn in MIGRATIONS: name = fn.__name__ if name not in already_applied: fn(db) db["_llm_migrations"].insert( { "name": name, "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), } ) already_applied.add(name) def ensure_migrations_table(db): if not db["_llm_migrations"].exists(): db["_llm_migrations"].create( { "name": str, "applied_at": str, }, pk="name", ) @migration def m001_initial(db): # Ensure the original table design exists, so other migrations can run if db["log"].exists(): # It needs to have the chat_id column if "chat_id" not in db["log"].columns_dict: db["log"].add_column("chat_id") return db["log"].create( { "provider": str, "system": str, "prompt": str, "chat_id": str, "response": str, "model": str, "timestamp": str, } ) @migration def m002_id_primary_key(db): db["log"].transform(pk="id") @migration def m003_chat_id_foreign_key(db): db["log"].transform(types={"chat_id": int}) db["log"].add_foreign_key("chat_id", "log", "id") @migration def m004_column_order(db): db["log"].transform( column_order=( "id", "model", "timestamp", "prompt", "system", "response", "chat_id", ) ) @migration def m004_drop_provider(db): db["log"].transform(drop=("provider",)) @migration def m005_debug(db): db["log"].add_column("debug", str) db["log"].add_column("duration_ms", int) @migration def m006_new_logs_table(db): columns = db["log"].columns_dict for column, type in ( ("options_json", str), ("prompt_json", str), ("response_json", str), ("reply_to_id", int), ): # It's possible people running development code like myself # might have accidentally created these columns already if column not in columns: db["log"].add_column(column, type) # Use .transform() to rename options and timestamp_utc, and set new order db["log"].transform( column_order=( "id", "model", "prompt", "system", "prompt_json", "options_json", "response", "response_json", "reply_to_id", "chat_id", "duration_ms", "timestamp_utc", ), rename={ "timestamp": "timestamp_utc", "options": "options_json", }, ) @migration def m007_finish_logs_table(db): db["log"].transform( drop={"debug"}, rename={"timestamp_utc": "datetime_utc"}, drop_foreign_keys=("chat_id",), ) with db.conn: db.execute("alter table log rename to logs") @migration def m008_reply_to_id_foreign_key(db): db["logs"].add_foreign_key("reply_to_id", "logs", "id") @migration def m008_fix_column_order_in_logs(db): # reply_to_id ended up at the end after foreign key added db["logs"].transform( column_order=( "id", "model", "prompt", "system", "prompt_json", "options_json", "response", "response_json", "reply_to_id", "chat_id", "duration_ms", "timestamp_utc", ), ) @migration def m009_delete_logs_table_if_empty(db): # We moved to a new table design, but we don't delete the table # if someone has put data in it if not db["logs"].count: db["logs"].drop() @migration def m010_create_new_log_tables(db): db["conversations"].create( { "id": str, "name": str, "model": str, }, pk="id", ) db["responses"].create( { "id": str, "model": str, "prompt": str, "system": str, "prompt_json": str, "options_json": str, "response": str, "response_json": str, "conversation_id": str, "duration_ms": int, "datetime_utc": str, }, pk="id", foreign_keys=(("conversation_id", "conversations", "id"),), ) @migration def m011_fts_for_responses(db): db["responses"].enable_fts(["prompt", "response"], create_triggers=True) @migration def m012_attachments_tables(db): db["attachments"].create( { "id": str, "type": str, "path": str, "url": str, "content": bytes, }, pk="id", ) db["prompt_attachments"].create( { "response_id": str, "attachment_id": str, "order": int, }, foreign_keys=( ("response_id", "responses", "id"), ("attachment_id", "attachments", "id"), ), pk=("response_id", "attachment_id"), ) @migration def m013_usage(db): db["responses"].add_column("input_tokens", int) db["responses"].add_column("output_tokens", int) db["responses"].add_column("token_details", str) @migration def m014_schemas(db): db["schemas"].create( { "id": str, "content": str, }, pk="id", ) db["responses"].add_column("schema_id", str, fk="schemas", fk_col="id") # Clean up SQL create table indentation db["responses"].transform() # These changes may have dropped the FTS configuration, fix that db["responses"].enable_fts( ["prompt", "response"], create_triggers=True, replace=True ) @migration def m015_fragments_tables(db): db["fragments"].create( { "id": int, "hash": str, "content": str, "datetime_utc": str, "source": str, }, pk="id", ) db["fragments"].create_index(["hash"], unique=True) db["fragment_aliases"].create( { "alias": str, "fragment_id": int, }, foreign_keys=(("fragment_id", "fragments", "id"),), pk="alias", ) db["prompt_fragments"].create( { "response_id": str, "fragment_id": int, "order": int, }, foreign_keys=( ("response_id", "responses", "id"), ("fragment_id", "fragments", "id"), ), pk=("response_id", "fragment_id"), ) db["system_fragments"].create( { "response_id": str, "fragment_id": int, "order": int, }, foreign_keys=( ("response_id", "responses", "id"), ("fragment_id", "fragments", "id"), ), pk=("response_id", "fragment_id"), ) @migration def m016_fragments_table_pks(db): # The same fragment can be attached to a response multiple times # https://github.com/simonw/llm/issues/863#issuecomment-2781720064 db["prompt_fragments"].transform(pk=("response_id", "fragment_id", "order")) db["system_fragments"].transform(pk=("response_id", "fragment_id", "order")) @migration def m017_tools_tables(db): db["tools"].create( { "id": int, "hash": str, "name": str, "description": str, "input_schema": str, }, pk="id", ) db["tools"].create_index(["hash"], unique=True) # Many-to-many relationship between tools and responses db["tool_responses"].create( { "tool_id": int, "response_id": str, }, foreign_keys=( ("tool_id", "tools", "id"), ("response_id", "responses", "id"), ), pk=("tool_id", "response_id"), ) # tool_calls and tool_results are one-to-many against responses db["tool_calls"].create( { "id": int, "response_id": str, "tool_id": int, "name": str, "arguments": str, "tool_call_id": str, }, pk="id", foreign_keys=( ("response_id", "responses", "id"), ("tool_id", "tools", "id"), ), ) db["tool_results"].create( { "id": int, "response_id": str, "tool_id": int, "name": str, "output": str, "tool_call_id": str, }, pk="id", foreign_keys=( ("response_id", "responses", "id"), ("tool_id", "tools", "id"), ), ) @migration def m017_tools_plugin(db): db["tools"].add_column("plugin") @migration def m018_tool_instances(db): # Used to track instances of Toolbox classes that may be # used multiple times by different tools db["tool_instances"].create( { "id": int, "plugin": str, "name": str, "arguments": str, }, pk="id", ) # We record which instance was used only on the results db["tool_results"].add_column("instance_id", fk="tool_instances") @migration def m019_resolved_model(db): # For models like gemini-1.5-flash-latest where we wish to record # the resolved model name in addition to the alias db["responses"].add_column("resolved_model", str) @migration def m020_tool_results_attachments(db): db["tool_results_attachments"].create( { "tool_result_id": int, "attachment_id": str, "order": int, }, foreign_keys=( ("tool_result_id", "tool_results", "id"), ("attachment_id", "attachments", "id"), ), pk=("tool_result_id", "attachment_id"), ) @migration def m021_tool_results_exception(db): db["tool_results"].add_column("exception", str) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/models.py0000644000175100001660000021435715046547145014440 0ustar00runnerdockerimport asyncio import base64 from condense_json import condense_json from dataclasses import dataclass, field import datetime from .errors import NeedsKeyException import hashlib import httpx from itertools import islice import re import time from types import MethodType from typing import ( Any, AsyncGenerator, AsyncIterator, Awaitable, Callable, Dict, Iterable, Iterator, List, Optional, Set, Union, get_type_hints, ) from .utils import ( ensure_fragment, ensure_tool, make_schema_id, mimetype_from_path, mimetype_from_string, token_usage_string, monotonic_ulid, ) from abc import ABC, abstractmethod import inspect import json from pydantic import BaseModel, ConfigDict, create_model CONVERSATION_NAME_LENGTH = 32 @dataclass class Usage: input: Optional[int] = None output: Optional[int] = None details: Optional[Dict[str, Any]] = None @dataclass class Attachment: type: Optional[str] = None path: Optional[str] = None url: Optional[str] = None content: Optional[bytes] = None _id: Optional[str] = None def id(self): # Hash of the binary content, or of '{"url": "https://..."}' for URL attachments if self._id is None: if self.content: self._id = hashlib.sha256(self.content).hexdigest() elif self.path: self._id = hashlib.sha256(open(self.path, "rb").read()).hexdigest() else: self._id = hashlib.sha256( json.dumps({"url": self.url}).encode("utf-8") ).hexdigest() return self._id def resolve_type(self): if self.type: return self.type # Derive it from path or url or content if self.path: return mimetype_from_path(self.path) if self.url: response = httpx.head(self.url) response.raise_for_status() return response.headers.get("content-type") if self.content: return mimetype_from_string(self.content) raise ValueError("Attachment has no type and no content to derive it from") def content_bytes(self): content = self.content if not content: if self.path: content = open(self.path, "rb").read() elif self.url: response = httpx.get(self.url) response.raise_for_status() content = response.content return content def base64_content(self): return base64.b64encode(self.content_bytes()).decode("utf-8") def __repr__(self): info = [f"" @classmethod def from_row(cls, row): return cls( _id=row["id"], type=row["type"], path=row["path"], url=row["url"], content=row["content"], ) @dataclass class Tool: name: str description: Optional[str] = None input_schema: Dict = field(default_factory=dict) implementation: Optional[Callable] = None plugin: Optional[str] = None # plugin tool came from, e.g. 'llm_tools_sqlite' def __post_init__(self): # Convert Pydantic model to JSON schema if needed self.input_schema = _ensure_dict_schema(self.input_schema) def hash(self): """Hash for tool based on its name, description and input schema (preserving key order)""" to_hash = { "name": self.name, "description": self.description, "input_schema": self.input_schema, } if self.plugin: to_hash["plugin"] = self.plugin return hashlib.sha256(json.dumps(to_hash).encode("utf-8")).hexdigest() @classmethod def function(cls, function, name=None, description=None): """ Turn a Python function into a Tool object by: - Extracting the function name - Using the function docstring for the Tool description - Building a Pydantic model for inputs by inspecting the function signature - Building a Pydantic model for the return value by using the function's return annotation """ if not name and function.__name__ == "": raise ValueError( "Cannot create a Tool from a lambda function without providing name=" ) return cls( name=name or function.__name__, description=description or function.__doc__ or None, input_schema=_get_arguments_input_schema(function, name), implementation=function, ) def _get_arguments_input_schema(function, name): signature = inspect.signature(function) type_hints = get_type_hints(function) fields = {} for param_name, param in signature.parameters.items(): if param_name == "self": continue # Determine the type annotation (default to string if missing) annotated_type = type_hints.get(param_name, str) # Handle default value if present; if there's no default, use '...' if param.default is inspect.Parameter.empty: fields[param_name] = (annotated_type, ...) else: fields[param_name] = (annotated_type, param.default) return create_model(f"{name}InputSchema", **fields) class Toolbox: name: Optional[str] = None instance_id: Optional[int] = None _blocked = ( "tools", "add_tool", "method_tools", "__init_subclass__", "prepare", "prepare_async", ) _extra_tools: List[Tool] = [] _config: Dict[str, Any] = {} _prepared: bool = False _async_prepared: bool = False def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) original_init = cls.__init__ def wrapped_init(self, *args, **kwargs): # Track args/kwargs passed to constructor in self._config # so we can serialize them to a database entry later on sig = inspect.signature(original_init) bound = sig.bind(self, *args, **kwargs) bound.apply_defaults() self._config = { name: value for name, value in bound.arguments.items() if name != "self" and sig.parameters[name].kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) } self._extra_tools = [] original_init(self, *args, **kwargs) cls.__init__ = wrapped_init @classmethod def method_tools(cls) -> List[Tool]: tools = [] for method_name in dir(cls): if method_name.startswith("_") or method_name in cls._blocked: continue method = getattr(cls, method_name) if callable(method): tool = Tool.function( method, name="{}_{}".format(cls.__name__, method_name), ) tools.append(tool) return tools def tools(self) -> Iterable[Tool]: "Returns an llm.Tool() for each class method, plus any extras registered with add_tool()" # method_tools() returns unbound methods, we need bound methods here: for name in dir(self): if name.startswith("_") or name in self._blocked: continue attr = getattr(self, name) if callable(attr): tool = Tool.function(attr, name=f"{self.__class__.__name__}_{name}") tool.plugin = getattr(self, "plugin", None) yield tool yield from self._extra_tools def add_tool( self, tool_or_function: Union[Tool, Callable[..., Any]], pass_self: bool = False ): "Add a tool to this toolbox" def _upgrade(fn): if pass_self: return MethodType(fn, self) return fn if isinstance(tool_or_function, Tool): self._extra_tools.append(tool_or_function) elif callable(tool_or_function): self._extra_tools.append(Tool.function(_upgrade(tool_or_function))) else: raise ValueError("Tool must be an instance of Tool or a callable function") def prepare(self): """ Over-ride this to perform setup (and .add_tool() calls) before the toolbox is used. Implement a similar prepare_async() method for async setup. """ pass async def prepare_async(self): """ Over-ride this to perform async setup (and .add_tool() calls) before the toolbox is used. """ pass @dataclass class ToolCall: name: str arguments: dict tool_call_id: Optional[str] = None @dataclass class ToolResult: name: str output: str attachments: List[Attachment] = field(default_factory=list) tool_call_id: Optional[str] = None instance: Optional[Toolbox] = None exception: Optional[Exception] = None @dataclass class ToolOutput: "Tool functions can return output with extra attachments" output: Optional[Union[str, dict, list, bool, int, float]] = None attachments: List[Attachment] = field(default_factory=list) ToolDef = Union[Tool, Toolbox, Callable[..., Any]] BeforeCallSync = Callable[[Optional[Tool], ToolCall], None] AfterCallSync = Callable[[Tool, ToolCall, ToolResult], None] BeforeCallAsync = Callable[[Optional[Tool], ToolCall], Union[None, Awaitable[None]]] AfterCallAsync = Callable[[Tool, ToolCall, ToolResult], Union[None, Awaitable[None]]] class CancelToolCall(Exception): pass @dataclass class Prompt: _prompt: Optional[str] model: "Model" fragments: Optional[List[str]] attachments: Optional[List[Attachment]] _system: Optional[str] system_fragments: Optional[List[str]] prompt_json: Optional[str] schema: Optional[Union[Dict, type[BaseModel]]] tools: List[Tool] tool_results: List[ToolResult] options: "Options" def __init__( self, prompt, model, *, fragments=None, attachments=None, system=None, system_fragments=None, prompt_json=None, options=None, schema=None, tools=None, tool_results=None, ): self._prompt = prompt self.model = model self.attachments = list(attachments or []) self.fragments = fragments or [] self._system = system self.system_fragments = system_fragments or [] self.prompt_json = prompt_json if schema and not isinstance(schema, dict) and issubclass(schema, BaseModel): schema = schema.model_json_schema() self.schema = schema self.tools = _wrap_tools(tools or []) self.tool_results = tool_results or [] self.options = options or {} @property def prompt(self): return "\n".join(self.fragments + ([self._prompt] if self._prompt else [])) @property def system(self): bits = [ bit.strip() for bit in (self.system_fragments + [self._system or ""]) if bit.strip() ] return "\n\n".join(bits) def _wrap_tools(tools: List[ToolDef]) -> List[Tool]: wrapped_tools = [] for tool in tools: if isinstance(tool, Tool): wrapped_tools.append(tool) elif isinstance(tool, Toolbox): wrapped_tools.extend(tool.tools()) elif callable(tool): wrapped_tools.append(Tool.function(tool)) else: raise ValueError(f"Invalid tool: {tool}") return wrapped_tools @dataclass class _BaseConversation: model: "_BaseModel" id: str = field(default_factory=lambda: str(monotonic_ulid()).lower()) name: Optional[str] = None responses: List["_BaseResponse"] = field(default_factory=list) tools: Optional[List[ToolDef]] = None chain_limit: Optional[int] = None @classmethod @abstractmethod def from_row(cls, row: Any) -> "_BaseConversation": raise NotImplementedError @dataclass class Conversation(_BaseConversation): before_call: Optional[BeforeCallSync] = None after_call: Optional[AfterCallSync] = None def prompt( self, prompt: Optional[str] = None, *, fragments: Optional[List[str]] = None, attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, system_fragments: Optional[List[str]] = None, stream: bool = True, key: Optional[str] = None, **options, ) -> "Response": return Response( Prompt( prompt, model=self.model, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, options=self.model.Options(**options), ), self.model, stream, conversation=self, key=key, ) def chain( self, prompt: Optional[str] = None, *, fragments: Optional[List[str]] = None, attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[str]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, chain_limit: Optional[int] = None, before_call: Optional[BeforeCallSync] = None, after_call: Optional[AfterCallSync] = None, key: Optional[str] = None, options: Optional[dict] = None, ) -> "ChainResponse": self.model._validate_attachments(attachments) return ChainResponse( Prompt( prompt, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, model=self.model, options=self.model.Options(**(options or {})), ), model=self.model, stream=stream, conversation=self, key=key, before_call=before_call or self.before_call, after_call=after_call or self.after_call, chain_limit=chain_limit if chain_limit is not None else self.chain_limit, ) @classmethod def from_row(cls, row): from llm import get_model return cls( model=get_model(row["model"]), id=row["id"], name=row["name"], ) def __repr__(self): count = len(self.responses) s = "s" if count == 1 else "" return f"<{self.__class__.__name__}: {self.id} - {count} response{s}" @dataclass class AsyncConversation(_BaseConversation): before_call: Optional[BeforeCallAsync] = None after_call: Optional[AfterCallAsync] = None def chain( self, prompt: Optional[str] = None, *, fragments: Optional[List[str]] = None, attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[str]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, chain_limit: Optional[int] = None, before_call: Optional[BeforeCallAsync] = None, after_call: Optional[AfterCallAsync] = None, key: Optional[str] = None, options: Optional[dict] = None, ) -> "AsyncChainResponse": self.model._validate_attachments(attachments) return AsyncChainResponse( Prompt( prompt, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, model=self.model, options=self.model.Options(**(options or {})), ), model=self.model, stream=stream, conversation=self, key=key, before_call=before_call or self.before_call, after_call=after_call or self.after_call, chain_limit=chain_limit if chain_limit is not None else self.chain_limit, ) def prompt( self, prompt: Optional[str] = None, *, fragments: Optional[List[str]] = None, attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, system_fragments: Optional[List[str]] = None, stream: bool = True, key: Optional[str] = None, **options, ) -> "AsyncResponse": return AsyncResponse( Prompt( prompt, model=self.model, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools, tool_results=tool_results, system_fragments=system_fragments, options=self.model.Options(**options), ), self.model, stream, conversation=self, key=key, ) def to_sync_conversation(self): return Conversation( model=self.model, id=self.id, name=self.name, responses=[], # Because we only use this in logging tools=self.tools, chain_limit=self.chain_limit, ) @classmethod def from_row(cls, row): from llm import get_async_model return cls( model=get_async_model(row["model"]), id=row["id"], name=row["name"], ) def __repr__(self): count = len(self.responses) s = "s" if count == 1 else "" return f"<{self.__class__.__name__}: {self.id} - {count} response{s}" FRAGMENT_SQL = """ select 'prompt' as fragment_type, fragments.content, pf."order" as ord from prompt_fragments pf join fragments on pf.fragment_id = fragments.id where pf.response_id = :response_id union all select 'system' as fragment_type, fragments.content, sf."order" as ord from system_fragments sf join fragments on sf.fragment_id = fragments.id where sf.response_id = :response_id order by fragment_type desc, ord asc; """ class _BaseResponse: """Base response class shared between sync and async responses""" id: str prompt: "Prompt" stream: bool resolved_model: Optional[str] = None conversation: Optional["_BaseConversation"] = None _key: Optional[str] = None _tool_calls: List[ToolCall] = [] def __init__( self, prompt: Prompt, model: "_BaseModel", stream: bool, conversation: Optional[_BaseConversation] = None, key: Optional[str] = None, ): self.id = str(monotonic_ulid()).lower() self.prompt = prompt self._prompt_json = None self.model = model self.stream = stream self._key = key self._chunks: List[str] = [] self._done = False self._tool_calls: List[ToolCall] = [] self.response_json: Optional[Dict[str, Any]] = None self.conversation = conversation self.attachments: List[Attachment] = [] self._start: Optional[float] = None self._end: Optional[float] = None self._start_utcnow: Optional[datetime.datetime] = None self.input_tokens: Optional[int] = None self.output_tokens: Optional[int] = None self.token_details: Optional[dict] = None self.done_callbacks: List[Callable] = [] if self.prompt.schema and not self.model.supports_schema: raise ValueError(f"{self.model} does not support schemas") if self.prompt.tools and not self.model.supports_tools: raise ValueError(f"{self.model} does not support tools") def add_tool_call(self, tool_call: ToolCall): self._tool_calls.append(tool_call) def set_usage( self, *, input: Optional[int] = None, output: Optional[int] = None, details: Optional[dict] = None, ): self.input_tokens = input self.output_tokens = output self.token_details = details def set_resolved_model(self, model_id: str): self.resolved_model = model_id @classmethod def from_row(cls, db, row, _async=False): from llm import get_model, get_async_model if _async: model = get_async_model(row["model"]) else: model = get_model(row["model"]) # Schema schema = None if row["schema_id"]: schema = json.loads(db["schemas"].get(row["schema_id"])["content"]) # Tool definitions and results for prompt tools = [ Tool( name=tool_row["name"], description=tool_row["description"], input_schema=json.loads(tool_row["input_schema"]), # In this case we don't have a reference to the actual Python code # but that's OK, we should not need it for prompts deserialized from DB implementation=None, plugin=tool_row["plugin"], ) for tool_row in db.query( """ select tools.* from tools join tool_responses on tools.id = tool_responses.tool_id where tool_responses.response_id = ? """, [row["id"]], ) ] tool_results = [ ToolResult( name=tool_results_row["name"], output=tool_results_row["output"], tool_call_id=tool_results_row["tool_call_id"], ) for tool_results_row in db.query( """ select * from tool_results where response_id = ? """, [row["id"]], ) ] all_fragments = list(db.query(FRAGMENT_SQL, {"response_id": row["id"]})) fragments = [ row["content"] for row in all_fragments if row["fragment_type"] == "prompt" ] system_fragments = [ row["content"] for row in all_fragments if row["fragment_type"] == "system" ] response = cls( model=model, prompt=Prompt( prompt=row["prompt"], model=model, fragments=fragments, attachments=[], system=row["system"], schema=schema, tools=tools, tool_results=tool_results, system_fragments=system_fragments, options=model.Options(**json.loads(row["options_json"])), ), stream=False, ) prompt_json = json.loads(row["prompt_json"] or "null") response.id = row["id"] response._prompt_json = prompt_json response.response_json = json.loads(row["response_json"] or "null") response._done = True response._chunks = [row["response"]] # Attachments response.attachments = [ Attachment.from_row(attachment_row) for attachment_row in db.query( """ select attachments.* from attachments join prompt_attachments on attachments.id = prompt_attachments.attachment_id where prompt_attachments.response_id = ? order by prompt_attachments."order" """, [row["id"]], ) ] # Tool calls response._tool_calls = [ ToolCall( name=tool_row["name"], arguments=json.loads(tool_row["arguments"]), tool_call_id=tool_row["tool_call_id"], ) for tool_row in db.query( """ select * from tool_calls where response_id = ? order by tool_call_id """, [row["id"]], ) ] return response def token_usage(self) -> str: return token_usage_string( self.input_tokens, self.output_tokens, self.token_details ) def log_to_db(self, db): conversation = self.conversation if not conversation: conversation = Conversation(model=self.model) db["conversations"].insert( { "id": conversation.id, "name": _conversation_name( self.prompt.prompt or self.prompt.system or "" ), "model": conversation.model.model_id, }, ignore=True, ) schema_id = None if self.prompt.schema: schema_id, schema_json = make_schema_id(self.prompt.schema) db["schemas"].insert({"id": schema_id, "content": schema_json}, ignore=True) response_id = self.id replacements = {} # Include replacements from previous responses for previous_response in conversation.responses[:-1]: for fragment in (previous_response.prompt.fragments or []) + ( previous_response.prompt.system_fragments or [] ): fragment_id = ensure_fragment(db, fragment) replacements[f"f:{fragment_id}"] = fragment replacements[f"r:{previous_response.id}"] = ( previous_response.text_or_raise() ) for i, fragment in enumerate(self.prompt.fragments): fragment_id = ensure_fragment(db, fragment) replacements[f"f{fragment_id}"] = fragment db["prompt_fragments"].insert( { "response_id": response_id, "fragment_id": fragment_id, "order": i, }, ) for i, fragment in enumerate(self.prompt.system_fragments): fragment_id = ensure_fragment(db, fragment) replacements[f"f{fragment_id}"] = fragment db["system_fragments"].insert( { "response_id": response_id, "fragment_id": fragment_id, "order": i, }, ) response_text = self.text_or_raise() replacements[f"r:{response_id}"] = response_text json_data = self.json() response = { "id": response_id, "model": self.model.model_id, "prompt": self.prompt._prompt, "system": self.prompt._system, "prompt_json": condense_json(self._prompt_json, replacements), "options_json": { key: value for key, value in dict(self.prompt.options).items() if value is not None }, "response": response_text, "response_json": condense_json(json_data, replacements), "conversation_id": conversation.id, "duration_ms": self.duration_ms(), "datetime_utc": self.datetime_utc(), "input_tokens": self.input_tokens, "output_tokens": self.output_tokens, "token_details": ( json.dumps(self.token_details) if self.token_details else None ), "schema_id": schema_id, "resolved_model": self.resolved_model, } db["responses"].insert(response) # Persist any attachments - loop through with index for index, attachment in enumerate(self.prompt.attachments): attachment_id = attachment.id() db["attachments"].insert( { "id": attachment_id, "type": attachment.resolve_type(), "path": attachment.path, "url": attachment.url, "content": attachment.content, }, replace=True, ) db["prompt_attachments"].insert( { "response_id": response_id, "attachment_id": attachment_id, "order": index, }, ) # Persist any tools, tool calls and tool results tool_ids_by_name = {} for tool in self.prompt.tools: tool_id = ensure_tool(db, tool) tool_ids_by_name[tool.name] = tool_id db["tool_responses"].insert( { "tool_id": tool_id, "response_id": response_id, } ) for tool_call in self.tool_calls(): # TODO Should be _or_raise() db["tool_calls"].insert( { "response_id": response_id, "tool_id": tool_ids_by_name.get(tool_call.name) or None, "name": tool_call.name, "arguments": json.dumps(tool_call.arguments), "tool_call_id": tool_call.tool_call_id, } ) for tool_result in self.prompt.tool_results: instance_id = None if tool_result.instance: try: if not tool_result.instance.instance_id: tool_result.instance.instance_id = ( db["tool_instances"] .insert( { "plugin": tool.plugin, "name": tool.name.split("_")[0], "arguments": json.dumps( tool_result.instance._config ), } ) .last_pk ) instance_id = tool_result.instance.instance_id except AttributeError: pass tool_result_id = ( db["tool_results"] .insert( { "response_id": response_id, "tool_id": tool_ids_by_name.get(tool_result.name) or None, "name": tool_result.name, "output": tool_result.output, "tool_call_id": tool_result.tool_call_id, "instance_id": instance_id, "exception": ( ( "{}: {}".format( tool_result.exception.__class__.__name__, str(tool_result.exception), ) ) if tool_result.exception else None ), } ) .last_pk ) # Persist attachments for tool results for index, attachment in enumerate(tool_result.attachments): attachment_id = attachment.id() db["attachments"].insert( { "id": attachment_id, "type": attachment.resolve_type(), "path": attachment.path, "url": attachment.url, "content": attachment.content, }, replace=True, ) db["tool_results_attachments"].insert( { "tool_result_id": tool_result_id, "attachment_id": attachment_id, "order": index, }, ) class Response(_BaseResponse): model: "Model" conversation: Optional["Conversation"] = None def on_done(self, callback): if not self._done: self.done_callbacks.append(callback) else: callback(self) def _on_done(self): for callback in self.done_callbacks: callback(self) def __str__(self) -> str: return self.text() def _force(self): if not self._done: list(self) def text(self) -> str: self._force() return "".join(self._chunks) def text_or_raise(self) -> str: return self.text() def execute_tool_calls( self, *, before_call: Optional[BeforeCallSync] = None, after_call: Optional[AfterCallSync] = None, ) -> List[ToolResult]: tool_results = [] tools_by_name = {tool.name: tool for tool in self.prompt.tools} # Run prepare() on all Toolbox instances that need it instances_to_prepare: list[Toolbox] = [] for tool_to_prep in tools_by_name.values(): inst = _get_instance(tool_to_prep.implementation) if isinstance(inst, Toolbox) and not getattr(inst, "_prepared", False): instances_to_prepare.append(inst) for inst in instances_to_prepare: inst.prepare() inst._prepared = True for tool_call in self.tool_calls(): tool: Optional[Tool] = tools_by_name.get(tool_call.name) # Tool could be None if the tool was not found in the prompt tools, # but we still call the before_call method: if before_call: try: cb_result = before_call(tool, tool_call) if inspect.isawaitable(cb_result): raise TypeError( "Asynchronous 'before_call' callback provided to a synchronous tool execution context. " "Please use an async chain/response or a synchronous callback." ) except CancelToolCall as ex: tool_results.append( ToolResult( name=tool_call.name, output="Cancelled: " + str(ex), tool_call_id=tool_call.tool_call_id, exception=ex, ) ) continue if tool is None: msg = 'tool "{}" does not exist'.format(tool_call.name) tool_results.append( ToolResult( name=tool_call.name, output="Error: " + msg, tool_call_id=tool_call.tool_call_id, exception=KeyError(msg), ) ) continue if not tool.implementation: raise ValueError( "No implementation available for tool: {}".format(tool_call.name) ) attachments = [] exception = None try: if asyncio.iscoroutinefunction(tool.implementation): result = asyncio.run(tool.implementation(**tool_call.arguments)) else: result = tool.implementation(**tool_call.arguments) if isinstance(result, ToolOutput): attachments = result.attachments result = result.output if not isinstance(result, str): result = json.dumps(result, default=repr) except Exception as ex: result = f"Error: {ex}" exception = ex tool_result_obj = ToolResult( name=tool_call.name, output=result, attachments=attachments, tool_call_id=tool_call.tool_call_id, instance=_get_instance(tool.implementation), exception=exception, ) if after_call: cb_result = after_call(tool, tool_call, tool_result_obj) if inspect.isawaitable(cb_result): raise TypeError( "Asynchronous 'after_call' callback provided to a synchronous tool execution context. " "Please use an async chain/response or a synchronous callback." ) tool_results.append(tool_result_obj) return tool_results def tool_calls(self) -> List[ToolCall]: self._force() return self._tool_calls def tool_calls_or_raise(self) -> List[ToolCall]: return self.tool_calls() def json(self) -> Optional[Dict[str, Any]]: self._force() return self.response_json def duration_ms(self) -> int: self._force() return int(((self._end or 0) - (self._start or 0)) * 1000) def datetime_utc(self) -> str: self._force() return self._start_utcnow.isoformat() if self._start_utcnow else "" def usage(self) -> Usage: self._force() return Usage( input=self.input_tokens, output=self.output_tokens, details=self.token_details, ) def __iter__(self) -> Iterator[str]: self._start = time.monotonic() self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) if self._done: yield from self._chunks return if isinstance(self.model, Model): for chunk in self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, ): assert chunk is not None yield chunk self._chunks.append(chunk) elif isinstance(self.model, KeyModel): for chunk in self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, key=self.model.get_key(self._key), ): assert chunk is not None yield chunk self._chunks.append(chunk) else: raise Exception("self.model must be a Model or KeyModel") if self.conversation: self.conversation.responses.append(self) self._end = time.monotonic() self._done = True self._on_done() def __repr__(self): text = "... not yet done ..." if self._done: text = "".join(self._chunks) return "".format(self.prompt.prompt, text) class AsyncResponse(_BaseResponse): model: "AsyncModel" conversation: Optional["AsyncConversation"] = None @classmethod def from_row(cls, db, row, _async=False): return super().from_row(db, row, _async=True) async def on_done(self, callback): if not self._done: self.done_callbacks.append(callback) else: if callable(callback): # Ensure we handle both sync and async callbacks correctly processed_callback = callback(self) if inspect.isawaitable(processed_callback): await processed_callback elif inspect.isawaitable(callback): await callback async def _on_done(self): for callback_func in self.done_callbacks: if callable(callback_func): processed_callback = callback_func(self) if inspect.isawaitable(processed_callback): await processed_callback elif inspect.isawaitable(callback_func): await callback_func async def execute_tool_calls( self, *, before_call: Optional[BeforeCallAsync] = None, after_call: Optional[AfterCallAsync] = None, ) -> List[ToolResult]: tool_calls_list = await self.tool_calls() tools_by_name = {tool.name: tool for tool in self.prompt.tools} # Run async prepare_async() on all Toolbox instances that need it instances_to_prepare: list[Toolbox] = [] for tool_to_prep in tools_by_name.values(): inst = _get_instance(tool_to_prep.implementation) if isinstance(inst, Toolbox) and not getattr( inst, "_async_prepared", False ): instances_to_prepare.append(inst) for inst in instances_to_prepare: await inst.prepare_async() inst._async_prepared = True indexed_results: List[tuple[int, ToolResult]] = [] async_tasks: List[asyncio.Task] = [] for idx, tc in enumerate(tool_calls_list): tool: Optional[Tool] = tools_by_name.get(tc.name) exception: Optional[Exception] = None if tool is None: output = f'Error: tool "{tc.name}" does not exist' exception = KeyError(tc.name) elif not tool.implementation: output = f'Error: tool "{tc.name}" has no implementation' exception = KeyError(tc.name) elif inspect.iscoroutinefunction(tool.implementation): async def run_async(tc=tc, tool=tool, idx=idx): # before_call inside the task if before_call: try: cb = before_call(tool, tc) if inspect.isawaitable(cb): await cb except CancelToolCall as ex: return idx, ToolResult( name=tc.name, output="Cancelled: " + str(ex), tool_call_id=tc.tool_call_id, exception=ex, ) exception = None attachments = [] try: result = await tool.implementation(**tc.arguments) if isinstance(result, ToolOutput): attachments.extend(result.attachments) result = result.output output = ( result if isinstance(result, str) else json.dumps(result, default=repr) ) except Exception as ex: output = f"Error: {ex}" exception = ex tr = ToolResult( name=tc.name, output=output, attachments=attachments, tool_call_id=tc.tool_call_id, instance=_get_instance(tool.implementation), exception=exception, ) # after_call inside the task if tool is not None and after_call: cb2 = after_call(tool, tc, tr) if inspect.isawaitable(cb2): await cb2 return idx, tr async_tasks.append(asyncio.create_task(run_async())) else: # Sync implementation: do hooks and call inline if before_call: try: cb = before_call(tool, tc) if inspect.isawaitable(cb): await cb except CancelToolCall as ex: indexed_results.append( ( idx, ToolResult( name=tc.name, output="Cancelled: " + str(ex), tool_call_id=tc.tool_call_id, exception=ex, ), ) ) continue exception = None attachments = [] if tool is None: output = f'Error: tool "{tc.name}" does not exist' exception = KeyError(tc.name) else: try: res = tool.implementation(**tc.arguments) if inspect.isawaitable(res): res = await res if isinstance(res, ToolOutput): attachments.extend(res.attachments) res = res.output output = ( res if isinstance(res, str) else json.dumps(res, default=repr) ) except Exception as ex: output = f"Error: {ex}" exception = ex tr = ToolResult( name=tc.name, output=output, attachments=attachments, tool_call_id=tc.tool_call_id, instance=_get_instance(tool.implementation), exception=exception, ) if tool is not None and after_call: cb2 = after_call(tool, tc, tr) if inspect.isawaitable(cb2): await cb2 indexed_results.append((idx, tr)) # Await all async tasks in parallel if async_tasks: indexed_results.extend(await asyncio.gather(*async_tasks)) # Reorder by original index indexed_results.sort(key=lambda x: x[0]) return [tr for _, tr in indexed_results] def __aiter__(self): self._start = time.monotonic() self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) if self._done: self._iter_chunks = list(self._chunks) # Make a copy for iteration return self async def __anext__(self) -> str: if self._done: if hasattr(self, "_iter_chunks") and self._iter_chunks: return self._iter_chunks.pop(0) raise StopAsyncIteration if not hasattr(self, "_generator"): if isinstance(self.model, AsyncModel): self._generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, ) elif isinstance(self.model, AsyncKeyModel): self._generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, key=self.model.get_key(self._key), ) else: raise ValueError("self.model must be an AsyncModel or AsyncKeyModel") try: chunk = await self._generator.__anext__() assert chunk is not None self._chunks.append(chunk) return chunk except StopAsyncIteration: if self.conversation: self.conversation.responses.append(self) self._end = time.monotonic() self._done = True if hasattr(self, "_generator"): del self._generator await self._on_done() raise async def _force(self): if not self._done: temp_chunks = [] async for chunk in self: temp_chunks.append(chunk) # This should populate self._chunks return self def text_or_raise(self) -> str: if not self._done: raise ValueError("Response not yet awaited") return "".join(self._chunks) async def text(self) -> str: await self._force() return "".join(self._chunks) async def tool_calls(self) -> List[ToolCall]: await self._force() return self._tool_calls def tool_calls_or_raise(self) -> List[ToolCall]: if not self._done: raise ValueError("Response not yet awaited") return self._tool_calls async def json(self) -> Optional[Dict[str, Any]]: await self._force() return self.response_json async def duration_ms(self) -> int: await self._force() return int(((self._end or 0) - (self._start or 0)) * 1000) async def datetime_utc(self) -> str: await self._force() return self._start_utcnow.isoformat() if self._start_utcnow else "" async def usage(self) -> Usage: await self._force() return Usage( input=self.input_tokens, output=self.output_tokens, details=self.token_details, ) def __await__(self): return self._force().__await__() async def to_sync_response(self) -> Response: await self._force() # This conversion might be tricky if the model is AsyncModel, # as Response expects a sync Model. For simplicity, we'll assume # the primary use case is data transfer after completion. # The model type on the new Response might need careful handling # if it's intended for further execution. # For now, let's assume self.model can be cast or is compatible. sync_model = self.model if not isinstance(self.model, (Model, KeyModel)): # This is a placeholder. A proper conversion or shared base might be needed # if the sync_response needs to be fully functional with its model. # For now, we pass the async model, which might limit what sync_response can do. pass response = Response( self.prompt, sync_model, # This might need adjustment based on how Model/AsyncModel relate self.stream, # conversation type needs to be compatible too. conversation=( self.conversation.to_sync_conversation() if self.conversation else None ), ) response.id = self.id response._chunks = list(self._chunks) # Copy chunks response._done = self._done response._end = self._end response._start = self._start response._start_utcnow = self._start_utcnow response.input_tokens = self.input_tokens response.output_tokens = self.output_tokens response.token_details = self.token_details response._prompt_json = self._prompt_json response.response_json = self.response_json response._tool_calls = list(self._tool_calls) response.attachments = list(self.attachments) response.resolved_model = self.resolved_model return response @classmethod def fake( cls, model: "AsyncModel", prompt: str, *attachments: List[Attachment], system: str, response: str, ): "Utility method to help with writing tests" response_obj = cls( model=model, prompt=Prompt( prompt, model=model, attachments=attachments, system=system, ), stream=False, ) response_obj._done = True response_obj._chunks = [response] return response_obj def __repr__(self): text = "... not yet awaited ..." if self._done: text = "".join(self._chunks) return "".format(self.prompt.prompt, text) class _BaseChainResponse: prompt: "Prompt" stream: bool conversation: Optional["_BaseConversation"] = None _key: Optional[str] = None def __init__( self, prompt: Prompt, model: "_BaseModel", stream: bool, conversation: _BaseConversation, key: Optional[str] = None, chain_limit: Optional[int] = 10, before_call: Optional[Union[BeforeCallSync, BeforeCallAsync]] = None, after_call: Optional[Union[AfterCallSync, AfterCallAsync]] = None, ): self.prompt = prompt self.model = model self.stream = stream self._key = key self._responses: List[Any] = [] self.conversation = conversation self.chain_limit = chain_limit self.before_call = before_call self.after_call = after_call def log_to_db(self, db): for response in self._responses: if isinstance(response, AsyncResponse): sync_response = asyncio.run(response.to_sync_response()) elif isinstance(response, Response): sync_response = response else: assert False, "Should have been a Response or AsyncResponse" sync_response.log_to_db(db) class ChainResponse(_BaseChainResponse): _responses: List["Response"] before_call: Optional[BeforeCallSync] = None after_call: Optional[AfterCallSync] = None def responses(self) -> Iterator[Response]: prompt = self.prompt count = 0 current_response: Optional[Response] = Response( prompt, self.model, self.stream, key=self._key, conversation=self.conversation, ) while current_response: count += 1 yield current_response self._responses.append(current_response) if self.chain_limit and count >= self.chain_limit: raise ValueError(f"Chain limit of {self.chain_limit} exceeded.") # This could raise llm.CancelToolCall: tool_results = current_response.execute_tool_calls( before_call=self.before_call, after_call=self.after_call ) attachments = [] for tool_result in tool_results: attachments.extend(tool_result.attachments) if tool_results: current_response = Response( Prompt( "", # Next prompt is empty, tools drive it self.model, tools=current_response.prompt.tools, tool_results=tool_results, options=self.prompt.options, attachments=attachments, ), self.model, stream=self.stream, key=self._key, conversation=self.conversation, ) else: current_response = None break def __iter__(self) -> Iterator[str]: for response_item in self.responses(): yield from response_item def text(self) -> str: return "".join(self) class AsyncChainResponse(_BaseChainResponse): _responses: List["AsyncResponse"] before_call: Optional[BeforeCallAsync] = None after_call: Optional[AfterCallAsync] = None async def responses(self) -> AsyncIterator[AsyncResponse]: prompt = self.prompt count = 0 current_response: Optional[AsyncResponse] = AsyncResponse( prompt, self.model, self.stream, key=self._key, conversation=self.conversation, ) while current_response: count += 1 yield current_response self._responses.append(current_response) if self.chain_limit and count >= self.chain_limit: raise ValueError(f"Chain limit of {self.chain_limit} exceeded.") # This could raise llm.CancelToolCall: tool_results = await current_response.execute_tool_calls( before_call=self.before_call, after_call=self.after_call ) if tool_results: attachments = [] for tool_result in tool_results: attachments.extend(tool_result.attachments) prompt = Prompt( "", self.model, tools=current_response.prompt.tools, tool_results=tool_results, options=self.prompt.options, attachments=attachments, ) current_response = AsyncResponse( prompt, self.model, stream=self.stream, key=self._key, conversation=self.conversation, ) else: current_response = None break async def __aiter__(self) -> AsyncIterator[str]: async for response_item in self.responses(): async for chunk in response_item: yield chunk async def text(self) -> str: all_chunks = [] async for chunk in self: all_chunks.append(chunk) return "".join(all_chunks) class Options(BaseModel): model_config = ConfigDict(extra="forbid") _Options = Options class _get_key_mixin: needs_key: Optional[str] = None key: Optional[str] = None key_env_var: Optional[str] = None def get_key(self, explicit_key: Optional[str] = None) -> Optional[str]: from llm import get_key if self.needs_key is None: # This model doesn't use an API key return None if self.key is not None: # Someone already set model.key='...' return self.key # Attempt to load a key using llm.get_key() key_value = get_key( explicit_key=explicit_key, key_alias=self.needs_key, env_var=self.key_env_var, ) if key_value: return key_value # Show a useful error message message = "No key found - add one using 'llm keys set {}'".format( self.needs_key ) if self.key_env_var: message += " or set the {} environment variable".format(self.key_env_var) raise NeedsKeyException(message) class _BaseModel(ABC, _get_key_mixin): model_id: str can_stream: bool = False attachment_types: Set = set() supports_schema = False supports_tools = False class Options(_Options): pass def _validate_attachments( self, attachments: Optional[List[Attachment]] = None ) -> None: if attachments and not self.attachment_types: raise ValueError("This model does not support attachments") for attachment in attachments or []: attachment_type = attachment.resolve_type() if attachment_type not in self.attachment_types: raise ValueError( f"This model does not support attachments of type '{attachment_type}', " f"only {', '.join(self.attachment_types)}" ) def __str__(self) -> str: return "{}{}: {}".format( self.__class__.__name__, " (async)" if isinstance(self, (AsyncModel, AsyncKeyModel)) else "", self.model_id, ) def __repr__(self) -> str: return f"<{str(self)}>" class _Model(_BaseModel): def conversation( self, tools: Optional[List[ToolDef]] = None, before_call: Optional[BeforeCallSync] = None, after_call: Optional[AfterCallSync] = None, chain_limit: Optional[int] = None, ) -> Conversation: return Conversation( model=self, tools=tools, before_call=before_call, after_call=after_call, chain_limit=chain_limit, ) def prompt( self, prompt: Optional[str] = None, *, fragments: Optional[List[str]] = None, attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[str]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, **options, ) -> Response: key_value = options.pop("key", None) self._validate_attachments(attachments) return Response( Prompt( prompt, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools, tool_results=tool_results, system_fragments=system_fragments, model=self, options=self.Options(**options), ), self, stream, key=key_value, ) def chain( self, prompt: Optional[str] = None, *, fragments: Optional[List[str]] = None, attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[str]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, before_call: Optional[BeforeCallSync] = None, after_call: Optional[AfterCallSync] = None, key: Optional[str] = None, options: Optional[dict] = None, ) -> ChainResponse: return self.conversation().chain( prompt=prompt, fragments=fragments, attachments=attachments, system=system, system_fragments=system_fragments, stream=stream, schema=schema, tools=tools, tool_results=tool_results, before_call=before_call, after_call=after_call, key=key, options=options, ) class Model(_Model): @abstractmethod def execute( self, prompt: Prompt, stream: bool, response: Response, conversation: Optional[Conversation], ) -> Iterator[str]: pass class KeyModel(_Model): @abstractmethod def execute( self, prompt: Prompt, stream: bool, response: Response, conversation: Optional[Conversation], key: Optional[str], ) -> Iterator[str]: pass class _AsyncModel(_BaseModel): def conversation( self, tools: Optional[List[ToolDef]] = None, before_call: Optional[BeforeCallAsync] = None, after_call: Optional[AfterCallAsync] = None, chain_limit: Optional[int] = None, ) -> AsyncConversation: return AsyncConversation( model=self, tools=tools, before_call=before_call, after_call=after_call, chain_limit=chain_limit, ) def prompt( self, prompt: Optional[str] = None, *, fragments: Optional[List[str]] = None, attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, system_fragments: Optional[List[str]] = None, stream: bool = True, **options, ) -> AsyncResponse: key_value = options.pop("key", None) self._validate_attachments(attachments) return AsyncResponse( Prompt( prompt, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools, tool_results=tool_results, system_fragments=system_fragments, model=self, options=self.Options(**options), ), self, stream, key=key_value, ) def chain( self, prompt: Optional[str] = None, *, fragments: Optional[List[str]] = None, attachments: Optional[List[Attachment]] = None, system: Optional[str] = None, system_fragments: Optional[List[str]] = None, stream: bool = True, schema: Optional[Union[dict, type[BaseModel]]] = None, tools: Optional[List[ToolDef]] = None, tool_results: Optional[List[ToolResult]] = None, before_call: Optional[BeforeCallAsync] = None, after_call: Optional[AfterCallAsync] = None, key: Optional[str] = None, options: Optional[dict] = None, ) -> AsyncChainResponse: return self.conversation().chain( prompt=prompt, fragments=fragments, attachments=attachments, system=system, system_fragments=system_fragments, stream=stream, schema=schema, tools=tools, tool_results=tool_results, before_call=before_call, after_call=after_call, key=key, options=options, ) class AsyncModel(_AsyncModel): @abstractmethod async def execute( self, prompt: Prompt, stream: bool, response: AsyncResponse, conversation: Optional[AsyncConversation], ) -> AsyncGenerator[str, None]: if False: # Ensure it's a generator type yield "" pass class AsyncKeyModel(_AsyncModel): @abstractmethod async def execute( self, prompt: Prompt, stream: bool, response: AsyncResponse, conversation: Optional[AsyncConversation], key: Optional[str], ) -> AsyncGenerator[str, None]: if False: # Ensure it's a generator type yield "" pass class EmbeddingModel(ABC, _get_key_mixin): model_id: str key: Optional[str] = None needs_key: Optional[str] = None key_env_var: Optional[str] = None supports_text: bool = True supports_binary: bool = False batch_size: Optional[int] = None def _check(self, item: Union[str, bytes]): if not self.supports_binary and isinstance(item, bytes): raise ValueError( "This model does not support binary data, only text strings" ) if not self.supports_text and isinstance(item, str): raise ValueError( "This model does not support text strings, only binary data" ) def embed(self, item: Union[str, bytes]) -> List[float]: "Embed a single text string or binary blob, return a list of floats" self._check(item) return next(iter(self.embed_batch([item]))) def embed_multi( self, items: Iterable[Union[str, bytes]], batch_size: Optional[int] = None ) -> Iterator[List[float]]: "Embed multiple items in batches according to the model batch_size" iter_items = iter(items) effective_batch_size = self.batch_size if batch_size is None else batch_size if (not self.supports_binary) or (not self.supports_text): def checking_iter(inner_items): for item_to_check in inner_items: self._check(item_to_check) yield item_to_check iter_items = checking_iter(items) if effective_batch_size is None: yield from self.embed_batch(iter_items) return while True: batch_items = list(islice(iter_items, effective_batch_size)) if not batch_items: break yield from self.embed_batch(batch_items) @abstractmethod def embed_batch(self, items: Iterable[Union[str, bytes]]) -> Iterator[List[float]]: """ Embed a batch of strings or blobs, return a list of lists of floats """ pass def __str__(self) -> str: return "{}: {}".format(self.__class__.__name__, self.model_id) def __repr__(self) -> str: return f"<{str(self)}>" @dataclass class ModelWithAliases: model: Model async_model: AsyncModel aliases: Set[str] def matches(self, query: str) -> bool: query_lower = query.lower() all_strings: List[str] = [] all_strings.extend(self.aliases) if self.model: all_strings.append(str(self.model)) if self.async_model: all_strings.append(str(self.async_model.model_id)) return any(query_lower in alias.lower() for alias in all_strings) @dataclass class EmbeddingModelWithAliases: model: EmbeddingModel aliases: Set[str] def matches(self, query: str) -> bool: query_lower = query.lower() all_strings: List[str] = [] all_strings.extend(self.aliases) all_strings.append(str(self.model)) return any(query_lower in alias.lower() for alias in all_strings) def _conversation_name(text): # Collapse whitespace, including newlines text = re.sub(r"\s+", " ", text) if len(text) <= CONVERSATION_NAME_LENGTH: return text return text[: CONVERSATION_NAME_LENGTH - 1] + "…" def _ensure_dict_schema(schema): """Convert a Pydantic model to a JSON schema dict if needed.""" if schema and not isinstance(schema, dict) and issubclass(schema, BaseModel): schema_dict = schema.model_json_schema() _remove_titles_recursively(schema_dict) return schema_dict return schema def _remove_titles_recursively(obj): """Recursively remove all 'title' fields from a nested dictionary.""" if isinstance(obj, dict): # Remove title if present obj.pop("title", None) # Recursively process all values for value in obj.values(): _remove_titles_recursively(value) elif isinstance(obj, list): # Process each item in lists for item in obj: _remove_titles_recursively(item) def _get_instance(implementation): if hasattr(implementation, "__self__"): return implementation.__self__ return None ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/plugins.py0000644000175100001660000000312215046547145014620 0ustar00runnerdockerimport importlib from importlib import metadata import os import pluggy import sys from . import hookspecs DEFAULT_PLUGINS = ( "llm.default_plugins.openai_models", "llm.default_plugins.default_tools", ) pm = pluggy.PluginManager("llm") pm.add_hookspecs(hookspecs) LLM_LOAD_PLUGINS = os.environ.get("LLM_LOAD_PLUGINS", None) _loaded = False def load_plugins(): global _loaded if _loaded: return _loaded = True if not hasattr(sys, "_called_from_test") and LLM_LOAD_PLUGINS is None: # Only load plugins if not running tests pm.load_setuptools_entrypoints("llm") # Load any plugins specified in LLM_LOAD_PLUGINS") if LLM_LOAD_PLUGINS is not None: for package_name in [ name for name in LLM_LOAD_PLUGINS.split(",") if name.strip() ]: try: distribution = metadata.distribution(package_name) # Updated call llm_entry_points = [ ep for ep in distribution.entry_points if ep.group == "llm" ] for entry_point in llm_entry_points: mod = entry_point.load() pm.register(mod, name=entry_point.name) # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) # type: ignore except metadata.PackageNotFoundError: sys.stderr.write(f"Plugin {package_name} could not be found\n") for plugin in DEFAULT_PLUGINS: mod = importlib.import_module(plugin) pm.register(mod, plugin) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/py.typed0000644000175100001660000000000015046547145014254 0ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/templates.py0000644000175100001660000000554515046547145015150 0ustar00runnerdockerfrom pydantic import BaseModel, ConfigDict import string from typing import Optional, Any, Dict, List, Tuple class AttachmentType(BaseModel): type: str value: str class Template(BaseModel): name: str prompt: Optional[str] = None system: Optional[str] = None attachments: Optional[List[str]] = None attachment_types: Optional[List[AttachmentType]] = None model: Optional[str] = None defaults: Optional[Dict[str, Any]] = None options: Optional[Dict[str, Any]] = None extract: Optional[bool] = None # For extracting fenced code blocks extract_last: Optional[bool] = None schema_object: Optional[dict] = None fragments: Optional[List[str]] = None system_fragments: Optional[List[str]] = None tools: Optional[List[str]] = None functions: Optional[str] = None model_config = ConfigDict(extra="forbid") class MissingVariables(Exception): pass def __init__(self, **data): super().__init__(**data) # Not a pydantic field to avoid YAML being able to set it # this controls if Python inline functions code is trusted self._functions_is_trusted = False def evaluate( self, input: str, params: Optional[Dict[str, Any]] = None ) -> Tuple[Optional[str], Optional[str]]: params = params or {} params["input"] = input if self.defaults: for k, v in self.defaults.items(): if k not in params: params[k] = v prompt: Optional[str] = None system: Optional[str] = None if not self.prompt: system = self.interpolate(self.system, params) prompt = input else: prompt = self.interpolate(self.prompt, params) system = self.interpolate(self.system, params) return prompt, system def vars(self) -> set: all_vars = set() for text in [self.prompt, self.system]: if not text: continue all_vars.update(self.extract_vars(string.Template(text))) return all_vars @classmethod def interpolate(cls, text: Optional[str], params: Dict[str, Any]) -> Optional[str]: if not text: return text # Confirm all variables in text are provided string_template = string.Template(text) vars = cls.extract_vars(string_template) missing = [p for p in vars if p not in params] if missing: raise cls.MissingVariables( "Missing variables: {}".format(", ".join(missing)) ) return string_template.substitute(**params) @staticmethod def extract_vars(string_template: string.Template) -> List[str]: return [ match.group("named") for match in string_template.pattern.finditer(string_template.template) if match.group("named") ] ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/tools.py0000644000175100001660000000212715046547145014303 0ustar00runnerdockerfrom datetime import datetime, timezone from importlib.metadata import version import time def llm_version() -> str: "Return the installed version of llm" return version("llm") def llm_time() -> dict: "Returns the current time, as local time and UTC" # Get current times utc_time = datetime.now(timezone.utc) local_time = datetime.now() # Get timezone information local_tz_name = time.tzname[time.localtime().tm_isdst] is_dst = bool(time.localtime().tm_isdst) # Calculate offset offset_seconds = -time.timezone if not is_dst else -time.altzone offset_hours = offset_seconds // 3600 offset_minutes = (offset_seconds % 3600) // 60 timezone_offset = ( f"UTC{'+' if offset_hours >= 0 else ''}{offset_hours:02d}:{offset_minutes:02d}" ) return { "utc_time": utc_time.strftime("%Y-%m-%d %H:%M:%S UTC"), "utc_time_iso": utc_time.isoformat(), "local_timezone": local_tz_name, "local_time": local_time.strftime("%Y-%m-%d %H:%M:%S"), "timezone_offset": timezone_offset, "is_dst": is_dst, } ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/llm/utils.py0000644000175100001660000005734615046547145014320 0ustar00runnerdockerimport click import hashlib import httpx import itertools import json import pathlib import puremagic import re import sqlite_utils import textwrap from typing import Any, List, Dict, Optional, Tuple, Type import os import threading import time from typing import Final from ulid import ULID MIME_TYPE_FIXES = { "audio/wave": "audio/wav", } class Fragment(str): def __new__(cls, content, *args, **kwargs): # For immutable classes like str, __new__ creates the string object return super().__new__(cls, content) def __init__(self, content, source=""): # Initialize our custom attributes self.source = source def id(self): return hashlib.sha256(self.encode("utf-8")).hexdigest() def mimetype_from_string(content) -> Optional[str]: try: type_ = puremagic.from_string(content, mime=True) return MIME_TYPE_FIXES.get(type_, type_) except puremagic.PureError: return None def mimetype_from_path(path) -> Optional[str]: try: type_ = puremagic.from_file(path, mime=True) return MIME_TYPE_FIXES.get(type_, type_) except puremagic.PureError: return None def dicts_to_table_string( headings: List[str], dicts: List[Dict[str, str]] ) -> List[str]: max_lengths = [len(h) for h in headings] # Compute maximum length for each column for d in dicts: for i, h in enumerate(headings): if h in d and len(str(d[h])) > max_lengths[i]: max_lengths[i] = len(str(d[h])) # Generate formatted table strings res = [] res.append(" ".join(h.ljust(max_lengths[i]) for i, h in enumerate(headings))) for d in dicts: row = [] for i, h in enumerate(headings): row.append(str(d.get(h, "")).ljust(max_lengths[i])) res.append(" ".join(row)) return res def remove_dict_none_values(d): """ Recursively remove keys with value of None or value of a dict that is all values of None """ if not isinstance(d, dict): return d new_dict = {} for key, value in d.items(): if value is not None: if isinstance(value, dict): nested = remove_dict_none_values(value) if nested: new_dict[key] = nested elif isinstance(value, list): new_dict[key] = [remove_dict_none_values(v) for v in value] else: new_dict[key] = value return new_dict class _LogResponse(httpx.Response): def iter_bytes(self, *args, **kwargs): for chunk in super().iter_bytes(*args, **kwargs): click.echo(chunk.decode(), err=True) yield chunk class _LogTransport(httpx.BaseTransport): def __init__(self, transport: httpx.BaseTransport): self.transport = transport def handle_request(self, request: httpx.Request) -> httpx.Response: response = self.transport.handle_request(request) return _LogResponse( status_code=response.status_code, headers=response.headers, stream=response.stream, extensions=response.extensions, ) def _no_accept_encoding(request: httpx.Request): request.headers.pop("accept-encoding", None) def _log_response(response: httpx.Response): request = response.request click.echo(f"Request: {request.method} {request.url}", err=True) click.echo(" Headers:", err=True) for key, value in request.headers.items(): if key.lower() == "authorization": value = "[...]" if key.lower() == "cookie": value = value.split("=")[0] + "=..." click.echo(f" {key}: {value}", err=True) click.echo(" Body:", err=True) try: request_body = json.loads(request.content) click.echo( textwrap.indent(json.dumps(request_body, indent=2), " "), err=True ) except json.JSONDecodeError: click.echo(textwrap.indent(request.content.decode(), " "), err=True) click.echo(f"Response: status_code={response.status_code}", err=True) click.echo(" Headers:", err=True) for key, value in response.headers.items(): if key.lower() == "set-cookie": value = value.split("=")[0] + "=..." click.echo(f" {key}: {value}", err=True) click.echo(" Body:", err=True) def logging_client() -> httpx.Client: return httpx.Client( transport=_LogTransport(httpx.HTTPTransport()), event_hooks={"request": [_no_accept_encoding], "response": [_log_response]}, ) def simplify_usage_dict(d): # Recursively remove keys with value 0 and empty dictionaries def remove_empty_and_zero(obj): if isinstance(obj, dict): cleaned = { k: remove_empty_and_zero(v) for k, v in obj.items() if v != 0 and v != {} } return {k: v for k, v in cleaned.items() if v is not None and v != {}} return obj return remove_empty_and_zero(d) or {} def token_usage_string(input_tokens, output_tokens, token_details) -> str: bits = [] if input_tokens is not None: bits.append(f"{format(input_tokens, ',')} input") if output_tokens is not None: bits.append(f"{format(output_tokens, ',')} output") if token_details: bits.append(json.dumps(token_details)) return ", ".join(bits) def extract_fenced_code_block(text: str, last: bool = False) -> Optional[str]: """ Extracts and returns Markdown fenced code block found in the given text. The function handles fenced code blocks that: - Use at least three backticks (`). - May include a language tag immediately after the opening backticks. - Use more than three backticks as long as the closing fence has the same number. If no fenced code block is found, the function returns None. Args: text (str): The input text to search for a fenced code block. last (bool): Extract the last code block if True, otherwise the first. Returns: Optional[str]: The content of the fenced code block, or None if not found. """ # Regex pattern to match fenced code blocks # - ^ or \n ensures that the fence is at the start of a line # - (`{3,}) captures the opening backticks (at least three) # - (\w+)? optionally captures the language tag # - \n matches the newline after the opening fence # - (.*?) non-greedy match for the code block content # - (?P=fence) ensures that the closing fence has the same number of backticks # - [ ]* allows for optional spaces between the closing fence and newline # - (?=\n|$) ensures that the closing fence is followed by a newline or end of string pattern = re.compile( r"""(?m)^(?P`{3,})(?P\w+)?\n(?P.*?)^(?P=fence)[ ]*(?=\n|$)""", re.DOTALL, ) matches = list(pattern.finditer(text)) if matches: match = matches[-1] if last else matches[0] return match.group("code") return None def make_schema_id(schema: dict) -> Tuple[str, str]: schema_json = json.dumps(schema, separators=(",", ":")) schema_id = hashlib.blake2b(schema_json.encode(), digest_size=16).hexdigest() return schema_id, schema_json def output_rows_as_json(rows, nl=False, compact=False, json_cols=()): """ Output rows as JSON - either newline-delimited or an array Parameters: - rows: Iterable of dictionaries to output - nl: Boolean, if True, use newline-delimited JSON - compact: Boolean, if True uses [{"...": "..."}\n {"...": "..."}] format - json_cols: Iterable of columns that contain JSON Yields: - Stream of strings to be output """ current_iter, next_iter = itertools.tee(rows, 2) next(next_iter, None) first = True for row, next_row in itertools.zip_longest(current_iter, next_iter): is_last = next_row is None for col in json_cols: row[col] = json.loads(row[col]) if nl: # Newline-delimited JSON: one JSON object per line yield json.dumps(row) elif compact: # Compact array format: [{"...": "..."}\n {"...": "..."}] yield "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar="[" if first else " ", serialized=json.dumps(row), maybecomma="," if not is_last else "", lastchar="]" if is_last else "", ) else: # Pretty-printed array format with indentation yield "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar="[\n" if first else "", serialized=textwrap.indent(json.dumps(row, indent=2), " "), maybecomma="," if not is_last else "", lastchar="\n]" if is_last else "", ) first = False if first and not nl: # We didn't output any rows, so yield the empty list yield "[]" def resolve_schema_input(db, schema_input, load_template): # schema_input might be JSON or a filepath or an ID or t:name if not schema_input: return if schema_input.strip().startswith("t:"): name = schema_input.strip()[2:] schema_object = None try: template = load_template(name) schema_object = template.schema_object except ValueError: raise click.ClickException("Invalid template: {}".format(name)) if not schema_object: raise click.ClickException("Template '{}' has no schema".format(name)) return template.schema_object if schema_input.strip().startswith("{"): try: return json.loads(schema_input) except ValueError: pass if " " in schema_input.strip() or "," in schema_input: # Treat it as schema DSL return schema_dsl(schema_input) # Is it a file on disk? path = pathlib.Path(schema_input) if path.exists(): try: return json.loads(path.read_text()) except ValueError: raise click.ClickException("Schema file contained invalid JSON") # Last attempt: is it an ID in the DB? try: row = db["schemas"].get(schema_input) return json.loads(row["content"]) except (sqlite_utils.db.NotFoundError, ValueError): raise click.BadParameter("Invalid schema") def schema_summary(schema: dict) -> str: """ Extract property names from a JSON schema and format them in a concise way that highlights the array/object structure. Args: schema (dict): A JSON schema dictionary Returns: str: A human-friendly summary of the schema structure """ if not schema or not isinstance(schema, dict): return "" schema_type = schema.get("type", "") if schema_type == "object": props = schema.get("properties", {}) prop_summaries = [] for name, prop_schema in props.items(): prop_type = prop_schema.get("type", "") if prop_type == "array": items = prop_schema.get("items", {}) items_summary = schema_summary(items) prop_summaries.append(f"{name}: [{items_summary}]") elif prop_type == "object": nested_summary = schema_summary(prop_schema) prop_summaries.append(f"{name}: {nested_summary}") else: prop_summaries.append(name) return "{" + ", ".join(prop_summaries) + "}" elif schema_type == "array": items = schema.get("items", {}) return schema_summary(items) return "" def schema_dsl(schema_dsl: str, multi: bool = False) -> Dict[str, Any]: """ Build a JSON schema from a concise schema string. Args: schema_dsl: A string representing a schema in the concise format. Can be comma-separated or newline-separated. multi: Boolean, return a schema for an "items" array of these Returns: A dictionary representing the JSON schema. """ # Type mapping dictionary type_mapping = { "int": "integer", "float": "number", "bool": "boolean", "str": "string", } # Initialize the schema dictionary with required elements json_schema: Dict[str, Any] = {"type": "object", "properties": {}, "required": []} # Check if the schema is newline-separated or comma-separated if "\n" in schema_dsl: fields = [field.strip() for field in schema_dsl.split("\n") if field.strip()] else: fields = [field.strip() for field in schema_dsl.split(",") if field.strip()] # Process each field for field in fields: # Extract field name, type, and description if ":" in field: field_info, description = field.split(":", 1) description = description.strip() else: field_info = field description = "" # Process field name and type field_parts = field_info.strip().split() field_name = field_parts[0].strip() # Default type is string field_type = "string" # If type is specified, use it if len(field_parts) > 1: type_indicator = field_parts[1].strip() if type_indicator in type_mapping: field_type = type_mapping[type_indicator] # Add field to properties json_schema["properties"][field_name] = {"type": field_type} # Add description if provided if description: json_schema["properties"][field_name]["description"] = description # Add field to required list json_schema["required"].append(field_name) if multi: return multi_schema(json_schema) else: return json_schema def multi_schema(schema: dict) -> dict: "Wrap JSON schema in an 'items': [] array" return { "type": "object", "properties": {"items": {"type": "array", "items": schema}}, "required": ["items"], } def find_unused_key(item: dict, key: str) -> str: 'Return unused key, e.g. for {"id": "1"} and key "id" returns "id_"' while key in item: key += "_" return key def truncate_string( text: str, max_length: int = 100, normalize_whitespace: bool = False, keep_end: bool = False, ) -> str: """ Truncate a string to a maximum length, with options to normalize whitespace and keep both start and end. Args: text: The string to truncate max_length: Maximum length of the result string normalize_whitespace: If True, replace all whitespace with a single space keep_end: If True, keep both beginning and end of string Returns: Truncated string """ if not text: return text if normalize_whitespace: text = re.sub(r"\s+", " ", text) if len(text) <= max_length: return text # Minimum sensible length for keep_end is 9 characters: "a... z" min_keep_end_length = 9 if keep_end and max_length >= min_keep_end_length: # Calculate how much text to keep at each end # Subtract 5 for the "... " separator cutoff = (max_length - 5) // 2 return text[:cutoff] + "... " + text[-cutoff:] else: # Fall back to simple truncation for very small max_length return text[: max_length - 3] + "..." def ensure_fragment(db, content): sql = """ insert into fragments (hash, content, datetime_utc, source) values (:hash, :content, datetime('now'), :source) on conflict(hash) do nothing """ hash_id = hashlib.sha256(content.encode("utf-8")).hexdigest() source = None if isinstance(content, Fragment): source = content.source with db.conn: db.execute(sql, {"hash": hash_id, "content": content, "source": source}) return list( db.query("select id from fragments where hash = :hash", {"hash": hash_id}) )[0]["id"] def ensure_tool(db, tool): sql = """ insert into tools (hash, name, description, input_schema, plugin) values (:hash, :name, :description, :input_schema, :plugin) on conflict(hash) do nothing """ with db.conn: db.execute( sql, { "hash": tool.hash(), "name": tool.name, "description": tool.description, "input_schema": json.dumps(tool.input_schema), "plugin": tool.plugin, }, ) return list( db.query("select id from tools where hash = :hash", {"hash": tool.hash()}) )[0]["id"] def maybe_fenced_code(content: str) -> str: "Return the content as a fenced code block if it looks like code" is_code = False if content.count("<") > 10: is_code = True if not is_code: # Are 90% of the lines under 120 chars? lines = content.splitlines() if len(lines) > 3: num_short = sum(1 for line in lines if len(line) < 120) if num_short / len(lines) > 0.9: is_code = True if is_code: # Find number of backticks not already present num_backticks = 3 while "`" * num_backticks in content: num_backticks += 1 # Add backticks content = ( "\n" + "`" * num_backticks + "\n" + content.strip() + "\n" + "`" * num_backticks ) return content _plugin_prefix_re = re.compile(r"^[a-zA-Z0-9_-]+:") def has_plugin_prefix(value: str) -> bool: "Check if value starts with alphanumeric prefix followed by a colon" return bool(_plugin_prefix_re.match(value)) def _parse_kwargs(arg_str: str) -> Dict[str, Any]: """Parse key=value pairs where each value is valid JSON.""" tokens = [] buf = [] depth = 0 in_string = False string_char = "" escape = False for ch in arg_str: if in_string: buf.append(ch) if escape: escape = False elif ch == "\\": escape = True elif ch == string_char: in_string = False else: if ch in "\"'": in_string = True string_char = ch buf.append(ch) elif ch in "{[(": depth += 1 buf.append(ch) elif ch in "}])": depth -= 1 buf.append(ch) elif ch == "," and depth == 0: tokens.append("".join(buf).strip()) buf = [] else: buf.append(ch) if buf: tokens.append("".join(buf).strip()) kwargs: Dict[str, Any] = {} for token in tokens: if not token: continue if "=" not in token: raise ValueError(f"Invalid keyword spec segment: '{token}'") key, value_str = token.split("=", 1) key = key.strip() value_str = value_str.strip() try: value = json.loads(value_str) except json.JSONDecodeError as e: raise ValueError(f"Value for '{key}' is not valid JSON: {value_str}") from e kwargs[key] = value return kwargs def instantiate_from_spec(class_map: Dict[str, Type], spec: str): """ Instantiate a class from a specification string with flexible argument formats. This function parses a specification string that defines a class name and its constructor arguments, then instantiates the class using the provided class mapping. The specification supports multiple argument formats for flexibility. Parameters ---------- class_map : Dict[str, Type] A mapping from class names (strings) to their corresponding class objects. Only classes present in this mapping can be instantiated. spec : str A specification string defining the class to instantiate and its arguments. Format: "ClassName" or "ClassName(arguments)" Supported argument formats: - Empty: ClassName() - calls constructor with no arguments - JSON object: ClassName({"key": "value", "other": 42}) - unpacked as **kwargs - Single JSON value: ClassName("hello") or ClassName([1,2,3]) - passed as single positional argument - Key-value pairs: ClassName(name="test", count=5, items=[1,2]) - parsed as individual kwargs where values must be valid JSON Returns ------- object An instance of the specified class, constructed with the parsed arguments. Raises ------ ValueError If the spec string format is invalid, if the class name is not found in class_map, if JSON parsing fails, or if argument parsing encounters errors. """ m = re.fullmatch(r"\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\((.*)\))?\s*$", spec) if not m: raise ValueError(f"Invalid spec string: '{spec}'") class_name, arg_body = m.group(1), (m.group(2) or "").strip() if class_name not in class_map: raise ValueError(f"Unknown class '{class_name}'") cls = class_map[class_name] # No arguments at all if arg_body == "": return cls() # Starts with { -> JSON object to kwargs if arg_body.lstrip().startswith("{"): try: kw = json.loads(arg_body) except json.JSONDecodeError as e: raise ValueError("Argument JSON object is not valid JSON") from e if not isinstance(kw, dict): raise ValueError("Top-level JSON must be an object when using {} form") return cls(**kw) # Starts with quote / number / [ / t f n for single positional JSON value if re.match(r'\s*(["\[\d\-]|true|false|null)', arg_body, re.I): try: positional_value = json.loads(arg_body) except json.JSONDecodeError as e: raise ValueError("Positional argument must be valid JSON") from e return cls(positional_value) # Otherwise treat as key=value pairs kwargs = _parse_kwargs(arg_body) return cls(**kwargs) NANOSECS_IN_MILLISECS = 1000000 TIMESTAMP_LEN = 6 RANDOMNESS_LEN = 10 _lock: Final = threading.Lock() _last: Optional[bytes] = None # 16-byte last produced ULID def monotonic_ulid() -> ULID: """ Return a ULID instance that is guaranteed to be *strictly larger* than every other ULID returned by this function inside the same process. It works the same way the reference JavaScript `monotonicFactory` does: * If the current call happens in the same millisecond as the previous one, the 80-bit randomness part is incremented by exactly one. * As soon as the system clock moves forward, a brand-new ULID with cryptographically secure randomness is generated. * If more than 2**80 ULIDs are requested within a single millisecond an `OverflowError` is raised (practically impossible). """ global _last now_ms = time.time_ns() // NANOSECS_IN_MILLISECS with _lock: # First call if _last is None: _last = _fresh(now_ms) return ULID(_last) # Decode timestamp from the last ULID we handed out last_ms = int.from_bytes(_last[:TIMESTAMP_LEN], "big") # If the millisecond is the same, increment the randomness if now_ms == last_ms: rand_int = int.from_bytes(_last[TIMESTAMP_LEN:], "big") + 1 if rand_int >= 1 << (RANDOMNESS_LEN * 8): raise OverflowError( "Randomness overflow: > 2**80 ULIDs requested " "in one millisecond!" ) randomness = rand_int.to_bytes(RANDOMNESS_LEN, "big") _last = _last[:TIMESTAMP_LEN] + randomness return ULID(_last) # New millisecond, start fresh _last = _fresh(now_ms) return ULID(_last) def _fresh(ms: int) -> bytes: """Build a brand-new 16-byte ULID for the given millisecond.""" timestamp = int.to_bytes(ms, TIMESTAMP_LEN, "big") randomness = os.urandom(RANDOMNESS_LEN) return timestamp + randomness ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1754975852.910846 llm-0.27.1/llm.egg-info/0000755000175100001660000000000015046547155014262 5ustar00runnerdocker././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975852.0 llm-0.27.1/llm.egg-info/PKG-INFO0000644000175100001660000007325215046547154015367 0ustar00runnerdockerMetadata-Version: 2.4 Name: llm Version: 0.27.1 Summary: CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine. Author: Simon Willison License-Expression: Apache-2.0 Project-URL: Homepage, https://github.com/simonw/llm Project-URL: Documentation, https://llm.datasette.io/ Project-URL: Issues, https://github.com/simonw/llm/issues Project-URL: CI, https://github.com/simonw/llm/actions Project-URL: Changelog, https://github.com/simonw/llm/releases Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: Intended Audience :: End Users/Desktop Classifier: Intended Audience :: Science/Research Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence Classifier: Topic :: Text Processing :: Linguistic Classifier: Topic :: Utilities Requires-Python: >=3.9 Description-Content-Type: text/markdown License-File: LICENSE Requires-Dist: click Requires-Dist: condense-json>=0.1.3 Requires-Dist: openai>=1.55.3 Requires-Dist: click-default-group>=1.2.3 Requires-Dist: sqlite-utils>=3.37 Requires-Dist: sqlite-migrate>=0.1a2 Requires-Dist: pydantic>=2.0.0 Requires-Dist: PyYAML Requires-Dist: pluggy Requires-Dist: python-ulid Requires-Dist: setuptools Requires-Dist: pip Requires-Dist: pyreadline3; sys_platform == "win32" Requires-Dist: puremagic Provides-Extra: test Requires-Dist: build; extra == "test" Requires-Dist: click<8.2.0; extra == "test" Requires-Dist: pytest; extra == "test" Requires-Dist: numpy; extra == "test" Requires-Dist: pytest-httpx>=0.33.0; extra == "test" Requires-Dist: pytest-asyncio; extra == "test" Requires-Dist: cogapp; extra == "test" Requires-Dist: mypy>=1.10.0; extra == "test" Requires-Dist: black>=25.1.0; extra == "test" Requires-Dist: pytest-recording; extra == "test" Requires-Dist: ruff; extra == "test" Requires-Dist: syrupy; extra == "test" Requires-Dist: types-click; extra == "test" Requires-Dist: types-PyYAML; extra == "test" Requires-Dist: types-setuptools; extra == "test" Requires-Dist: llm-echo==0.3a3; extra == "test" Dynamic: license-file # LLM [![GitHub repo](https://img.shields.io/badge/github-repo-green)](https://github.com/simonw/llm) [![PyPI](https://img.shields.io/pypi/v/llm.svg)](https://pypi.org/project/llm/) [![Changelog](https://img.shields.io/github/v/release/simonw/llm?include_prereleases&label=changelog)](https://llm.datasette.io/en/stable/changelog.html) [![Tests](https://github.com/simonw/llm/workflows/Test/badge.svg)](https://github.com/simonw/llm/actions?query=workflow%3ATest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm/blob/main/LICENSE) [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Meta’s Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). With LLM you can: - [Run prompts from the command-line](https://llm.datasette.io/en/stable/usage.html#usage-executing-prompts) - [Store prompts and responses in SQLite](https://llm.datasette.io/en/stable/logging.html#logging) - [Generate and store embeddings](https://llm.datasette.io/en/stable/embeddings/index.html#embeddings) - [Extract structured content from text and images](https://llm.datasette.io/en/stable/schemas.html#schemas) - [Grant models the ability to execute tools](https://llm.datasette.io/en/stable/tools.html#tools) - … and much, much more ## Quick start First, install LLM using `pip` or Homebrew or `pipx` or `uv`: ```bash pip install llm ``` Or with Homebrew (see [warning note](https://llm.datasette.io/en/stable/setup.html#homebrew-warning)): ```bash brew install llm ``` Or with [pipx](https://pypa.github.io/pipx/): ```bash pipx install llm ``` Or with [uv](https://docs.astral.sh/uv/guides/tools/) ```bash uv tool install llm ``` If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash # Paste your OpenAI API key into this llm keys set openai # Run a prompt (with the default gpt-4o-mini model) llm "Ten fun names for a pet pelican" # Extract text from an image llm "extract text" -a scanned-document.jpg # Use a system prompt against a file cat myfile.py | llm -s "Explain this code" ``` Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic](https://console.anthropic.com/) with their respective plugins: ```bash llm install llm-gemini llm keys set gemini # Paste Gemini API key here llm -m gemini-2.0-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here llm -m claude-4-opus 'Impress me with wild facts about turnips' ``` You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/): ```bash # Install the plugin llm install llm-ollama # Download and run a prompt against the Orca Mini 7B model ollama pull llama3.2:latest llm -m llama3.2:latest 'What is the capital of France?' ``` To start [an interactive chat](https://llm.datasette.io/en/stable/usage.html#usage-chat) with a model, use `llm chat`: ```bash llm chat -m gpt-4.1 ``` ```default Chatting with gpt-4.1 Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt. Type '!fragment [ ...]' to insert one or more fragments > Tell me a joke about a pelican Why don't pelicans like to tip waiters? Because they always have a big bill! ``` More background on this project: - [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) - [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) - [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) - [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) - [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) - [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) - [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents * [Setup](https://llm.datasette.io/en/stable/setup.html) * [Installation](https://llm.datasette.io/en/stable/setup.html#installation) * [Upgrading to the latest version](https://llm.datasette.io/en/stable/setup.html#upgrading-to-the-latest-version) * [Using uvx](https://llm.datasette.io/en/stable/setup.html#using-uvx) * [A note about Homebrew and PyTorch](https://llm.datasette.io/en/stable/setup.html#a-note-about-homebrew-and-pytorch) * [Installing plugins](https://llm.datasette.io/en/stable/setup.html#installing-plugins) * [API key management](https://llm.datasette.io/en/stable/setup.html#api-key-management) * [Saving and using stored keys](https://llm.datasette.io/en/stable/setup.html#saving-and-using-stored-keys) * [Passing keys using the –key option](https://llm.datasette.io/en/stable/setup.html#passing-keys-using-the-key-option) * [Keys in environment variables](https://llm.datasette.io/en/stable/setup.html#keys-in-environment-variables) * [Configuration](https://llm.datasette.io/en/stable/setup.html#configuration) * [Setting a custom default model](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-default-model) * [Setting a custom directory location](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-directory-location) * [Turning SQLite logging on and off](https://llm.datasette.io/en/stable/setup.html#turning-sqlite-logging-on-and-off) * [Usage](https://llm.datasette.io/en/stable/usage.html) * [Executing a prompt](https://llm.datasette.io/en/stable/usage.html#executing-a-prompt) * [Model options](https://llm.datasette.io/en/stable/usage.html#model-options) * [Attachments](https://llm.datasette.io/en/stable/usage.html#attachments) * [System prompts](https://llm.datasette.io/en/stable/usage.html#system-prompts) * [Tools](https://llm.datasette.io/en/stable/usage.html#tools) * [Extracting fenced code blocks](https://llm.datasette.io/en/stable/usage.html#extracting-fenced-code-blocks) * [Schemas](https://llm.datasette.io/en/stable/usage.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/usage.html#fragments) * [Continuing a conversation](https://llm.datasette.io/en/stable/usage.html#continuing-a-conversation) * [Tips for using LLM with Bash or Zsh](https://llm.datasette.io/en/stable/usage.html#tips-for-using-llm-with-bash-or-zsh) * [Completion prompts](https://llm.datasette.io/en/stable/usage.html#completion-prompts) * [Starting an interactive chat](https://llm.datasette.io/en/stable/usage.html#starting-an-interactive-chat) * [Listing available models](https://llm.datasette.io/en/stable/usage.html#listing-available-models) * [Setting default options for models](https://llm.datasette.io/en/stable/usage.html#setting-default-options-for-models) * [OpenAI models](https://llm.datasette.io/en/stable/openai-models.html) * [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration) * [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models) * [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features) * [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models) * [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models) * [Adding more OpenAI models](https://llm.datasette.io/en/stable/openai-models.html#adding-more-openai-models) * [Other models](https://llm.datasette.io/en/stable/other-models.html) * [Installing and using a local model](https://llm.datasette.io/en/stable/other-models.html#installing-and-using-a-local-model) * [OpenAI-compatible models](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models) * [Extra HTTP headers](https://llm.datasette.io/en/stable/other-models.html#extra-http-headers) * [Tools](https://llm.datasette.io/en/stable/tools.html) * [How tools work](https://llm.datasette.io/en/stable/tools.html#how-tools-work) * [Trying out tools](https://llm.datasette.io/en/stable/tools.html#trying-out-tools) * [LLM’s implementation of tools](https://llm.datasette.io/en/stable/tools.html#llm-s-implementation-of-tools) * [Default tools](https://llm.datasette.io/en/stable/tools.html#default-tools) * [Tips for implementing tools](https://llm.datasette.io/en/stable/tools.html#tips-for-implementing-tools) * [Schemas](https://llm.datasette.io/en/stable/schemas.html) * [Schemas tutorial](https://llm.datasette.io/en/stable/schemas.html#schemas-tutorial) * [Getting started with dogs](https://llm.datasette.io/en/stable/schemas.html#getting-started-with-dogs) * [Extracting people from a news articles](https://llm.datasette.io/en/stable/schemas.html#extracting-people-from-a-news-articles) * [Using JSON schemas](https://llm.datasette.io/en/stable/schemas.html#using-json-schemas) * [Ways to specify a schema](https://llm.datasette.io/en/stable/schemas.html#ways-to-specify-a-schema) * [Concise LLM schema syntax](https://llm.datasette.io/en/stable/schemas.html#concise-llm-schema-syntax) * [Saving reusable schemas in templates](https://llm.datasette.io/en/stable/schemas.html#saving-reusable-schemas-in-templates) * [Browsing logged JSON objects created using schemas](https://llm.datasette.io/en/stable/schemas.html#browsing-logged-json-objects-created-using-schemas) * [Templates](https://llm.datasette.io/en/stable/templates.html) * [Getting started with –save](https://llm.datasette.io/en/stable/templates.html#getting-started-with-save) * [Using a template](https://llm.datasette.io/en/stable/templates.html#using-a-template) * [Listing available templates](https://llm.datasette.io/en/stable/templates.html#listing-available-templates) * [Templates as YAML files](https://llm.datasette.io/en/stable/templates.html#templates-as-yaml-files) * [System prompts](https://llm.datasette.io/en/stable/templates.html#system-prompts) * [Fragments](https://llm.datasette.io/en/stable/templates.html#fragments) * [Options](https://llm.datasette.io/en/stable/templates.html#options) * [Tools](https://llm.datasette.io/en/stable/templates.html#tools) * [Schemas](https://llm.datasette.io/en/stable/templates.html#schemas) * [Additional template variables](https://llm.datasette.io/en/stable/templates.html#additional-template-variables) * [Specifying default parameters](https://llm.datasette.io/en/stable/templates.html#specifying-default-parameters) * [Configuring code extraction](https://llm.datasette.io/en/stable/templates.html#configuring-code-extraction) * [Setting a default model for a template](https://llm.datasette.io/en/stable/templates.html#setting-a-default-model-for-a-template) * [Template loaders from plugins](https://llm.datasette.io/en/stable/templates.html#template-loaders-from-plugins) * [Fragments](https://llm.datasette.io/en/stable/fragments.html) * [Using fragments in a prompt](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-a-prompt) * [Using fragments in chat](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-chat) * [Browsing fragments](https://llm.datasette.io/en/stable/fragments.html#browsing-fragments) * [Setting aliases for fragments](https://llm.datasette.io/en/stable/fragments.html#setting-aliases-for-fragments) * [Viewing fragments in your logs](https://llm.datasette.io/en/stable/fragments.html#viewing-fragments-in-your-logs) * [Using fragments from plugins](https://llm.datasette.io/en/stable/fragments.html#using-fragments-from-plugins) * [Listing available fragment prefixes](https://llm.datasette.io/en/stable/fragments.html#listing-available-fragment-prefixes) * [Model aliases](https://llm.datasette.io/en/stable/aliases.html) * [Listing aliases](https://llm.datasette.io/en/stable/aliases.html#listing-aliases) * [Adding a new alias](https://llm.datasette.io/en/stable/aliases.html#adding-a-new-alias) * [Removing an alias](https://llm.datasette.io/en/stable/aliases.html#removing-an-alias) * [Viewing the aliases file](https://llm.datasette.io/en/stable/aliases.html#viewing-the-aliases-file) * [Embeddings](https://llm.datasette.io/en/stable/embeddings/index.html) * [Embedding with the CLI](https://llm.datasette.io/en/stable/embeddings/cli.html) * [llm embed](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed) * [llm embed-multi](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-multi) * [llm similar](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-similar) * [llm embed-models](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-models) * [llm collections list](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-list) * [llm collections delete](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-delete) * [Using embeddings from Python](https://llm.datasette.io/en/stable/embeddings/python-api.html) * [Working with collections](https://llm.datasette.io/en/stable/embeddings/python-api.html#working-with-collections) * [Retrieving similar items](https://llm.datasette.io/en/stable/embeddings/python-api.html#retrieving-similar-items) * [SQL schema](https://llm.datasette.io/en/stable/embeddings/python-api.html#sql-schema) * [Writing plugins to add new embedding models](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html) * [Embedding binary content](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#embedding-binary-content) * [Embedding storage format](https://llm.datasette.io/en/stable/embeddings/storage.html) * [Plugins](https://llm.datasette.io/en/stable/plugins/index.html) * [Installing plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html) * [Listing installed plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#listing-installed-plugins) * [Running with a subset of plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#running-with-a-subset-of-plugins) * [Plugin directory](https://llm.datasette.io/en/stable/plugins/directory.html) * [Local models](https://llm.datasette.io/en/stable/plugins/directory.html#local-models) * [Remote APIs](https://llm.datasette.io/en/stable/plugins/directory.html#remote-apis) * [Tools](https://llm.datasette.io/en/stable/plugins/directory.html#tools) * [Fragments and template loaders](https://llm.datasette.io/en/stable/plugins/directory.html#fragments-and-template-loaders) * [Embedding models](https://llm.datasette.io/en/stable/plugins/directory.html#embedding-models) * [Extra commands](https://llm.datasette.io/en/stable/plugins/directory.html#extra-commands) * [Just for fun](https://llm.datasette.io/en/stable/plugins/directory.html#just-for-fun) * [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html) * [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli) * [register_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register) * [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register) * [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register) * [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register) * [register_fragment_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-fragment-loaders-register) * [Developing a model plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html) * [The initial structure of the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#the-initial-structure-of-the-plugin) * [Installing your plugin to try it out](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#installing-your-plugin-to-try-it-out) * [Building the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#building-the-markov-chain) * [Executing the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#executing-the-markov-chain) * [Adding that to the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-that-to-the-plugin) * [Understanding execute()](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#understanding-execute) * [Prompts and responses are logged to the database](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#prompts-and-responses-are-logged-to-the-database) * [Adding options](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-options) * [Distributing your plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#distributing-your-plugin) * [GitHub repositories](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#github-repositories) * [Publishing plugins to PyPI](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#publishing-plugins-to-pypi) * [Adding metadata](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-metadata) * [What to do if it breaks](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#what-to-do-if-it-breaks) * [Advanced model plugins](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html) * [Tip: lazily load expensive dependencies](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tip-lazily-load-expensive-dependencies) * [Models that accept API keys](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#models-that-accept-api-keys) * [Async models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#async-models) * [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas) * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) * [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names) * [LLM_RAISE_ERRORS](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#llm-raise-errors) * [Utility functions for plugins](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html) * [llm.get_key()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-get-key) * [llm.user_dir()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-user-dir) * [llm.ModelError](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-modelerror) * [Response.fake()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#response-fake) * [Python API](https://llm.datasette.io/en/stable/python-api.html) * [Basic prompt execution](https://llm.datasette.io/en/stable/python-api.html#basic-prompt-execution) * [System prompts](https://llm.datasette.io/en/stable/python-api.html#system-prompts) * [Attachments](https://llm.datasette.io/en/stable/python-api.html#attachments) * [Tools](https://llm.datasette.io/en/stable/python-api.html#tools) * [Schemas](https://llm.datasette.io/en/stable/python-api.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/python-api.html#fragments) * [Model options](https://llm.datasette.io/en/stable/python-api.html#model-options) * [Passing an API key](https://llm.datasette.io/en/stable/python-api.html#passing-an-api-key) * [Models from plugins](https://llm.datasette.io/en/stable/python-api.html#models-from-plugins) * [Accessing the underlying JSON](https://llm.datasette.io/en/stable/python-api.html#accessing-the-underlying-json) * [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage) * [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses) * [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models) * [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async) * [Tool use for async models](https://llm.datasette.io/en/stable/python-api.html#tool-use-for-async-models) * [Conversations](https://llm.datasette.io/en/stable/python-api.html#conversations) * [Conversations using tools](https://llm.datasette.io/en/stable/python-api.html#conversations-using-tools) * [Listing models](https://llm.datasette.io/en/stable/python-api.html#listing-models) * [Running code when a response has completed](https://llm.datasette.io/en/stable/python-api.html#running-code-when-a-response-has-completed) * [Other functions](https://llm.datasette.io/en/stable/python-api.html#other-functions) * [set_alias(alias, model_id)](https://llm.datasette.io/en/stable/python-api.html#set-alias-alias-model-id) * [remove_alias(alias)](https://llm.datasette.io/en/stable/python-api.html#remove-alias-alias) * [set_default_model(alias)](https://llm.datasette.io/en/stable/python-api.html#set-default-model-alias) * [get_default_model()](https://llm.datasette.io/en/stable/python-api.html#get-default-model) * [set_default_embedding_model(alias) and get_default_embedding_model()](https://llm.datasette.io/en/stable/python-api.html#set-default-embedding-model-alias-and-get-default-embedding-model) * [Logging to SQLite](https://llm.datasette.io/en/stable/logging.html) * [Viewing the logs](https://llm.datasette.io/en/stable/logging.html#viewing-the-logs) * [-s/–short mode](https://llm.datasette.io/en/stable/logging.html#s-short-mode) * [Logs for a conversation](https://llm.datasette.io/en/stable/logging.html#logs-for-a-conversation) * [Searching the logs](https://llm.datasette.io/en/stable/logging.html#searching-the-logs) * [Filtering past a specific ID](https://llm.datasette.io/en/stable/logging.html#filtering-past-a-specific-id) * [Filtering by model](https://llm.datasette.io/en/stable/logging.html#filtering-by-model) * [Filtering by prompts that used specific fragments](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-fragments) * [Filtering by prompts that used specific tools](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-tools) * [Browsing data collected using schemas](https://llm.datasette.io/en/stable/logging.html#browsing-data-collected-using-schemas) * [Browsing logs using Datasette](https://llm.datasette.io/en/stable/logging.html#browsing-logs-using-datasette) * [Backing up your database](https://llm.datasette.io/en/stable/logging.html#backing-up-your-database) * [SQL schema](https://llm.datasette.io/en/stable/logging.html#sql-schema) * [Related tools](https://llm.datasette.io/en/stable/related-tools.html) * [strip-tags](https://llm.datasette.io/en/stable/related-tools.html#strip-tags) * [ttok](https://llm.datasette.io/en/stable/related-tools.html#ttok) * [Symbex](https://llm.datasette.io/en/stable/related-tools.html#symbex) * [CLI reference](https://llm.datasette.io/en/stable/help.html) * [llm –help](https://llm.datasette.io/en/stable/help.html#llm-help) * [llm prompt –help](https://llm.datasette.io/en/stable/help.html#llm-prompt-help) * [llm chat –help](https://llm.datasette.io/en/stable/help.html#llm-chat-help) * [llm keys –help](https://llm.datasette.io/en/stable/help.html#llm-keys-help) * [llm logs –help](https://llm.datasette.io/en/stable/help.html#llm-logs-help) * [llm models –help](https://llm.datasette.io/en/stable/help.html#llm-models-help) * [llm templates –help](https://llm.datasette.io/en/stable/help.html#llm-templates-help) * [llm schemas –help](https://llm.datasette.io/en/stable/help.html#llm-schemas-help) * [llm tools –help](https://llm.datasette.io/en/stable/help.html#llm-tools-help) * [llm aliases –help](https://llm.datasette.io/en/stable/help.html#llm-aliases-help) * [llm fragments –help](https://llm.datasette.io/en/stable/help.html#llm-fragments-help) * [llm plugins –help](https://llm.datasette.io/en/stable/help.html#llm-plugins-help) * [llm install –help](https://llm.datasette.io/en/stable/help.html#llm-install-help) * [llm uninstall –help](https://llm.datasette.io/en/stable/help.html#llm-uninstall-help) * [llm embed –help](https://llm.datasette.io/en/stable/help.html#llm-embed-help) * [llm embed-multi –help](https://llm.datasette.io/en/stable/help.html#llm-embed-multi-help) * [llm similar –help](https://llm.datasette.io/en/stable/help.html#llm-similar-help) * [llm embed-models –help](https://llm.datasette.io/en/stable/help.html#llm-embed-models-help) * [llm collections –help](https://llm.datasette.io/en/stable/help.html#llm-collections-help) * [llm openai –help](https://llm.datasette.io/en/stable/help.html#llm-openai-help) * [Contributing](https://llm.datasette.io/en/stable/contributing.html) * [Updating recorded HTTP API interactions and associated snapshots](https://llm.datasette.io/en/stable/contributing.html#updating-recorded-http-api-interactions-and-associated-snapshots) * [Debugging tricks](https://llm.datasette.io/en/stable/contributing.html#debugging-tricks) * [Documentation](https://llm.datasette.io/en/stable/contributing.html#documentation) * [Release process](https://llm.datasette.io/en/stable/contributing.html#release-process) * [Changelog](https://llm.datasette.io/en/stable/changelog.html) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975852.0 llm-0.27.1/llm.egg-info/SOURCES.txt0000644000175100001660000000103215046547154016141 0ustar00runnerdockerLICENSE MANIFEST.in README.md pyproject.toml llm/__init__.py llm/__main__.py llm/cli.py llm/embeddings.py llm/embeddings_migrations.py llm/errors.py llm/hookspecs.py llm/migrations.py llm/models.py llm/plugins.py llm/py.typed llm/templates.py llm/tools.py llm/utils.py llm.egg-info/PKG-INFO llm.egg-info/SOURCES.txt llm.egg-info/dependency_links.txt llm.egg-info/entry_points.txt llm.egg-info/requires.txt llm.egg-info/top_level.txt llm/default_plugins/__init__.py llm/default_plugins/default_tools.py llm/default_plugins/openai_models.py././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975852.0 llm-0.27.1/llm.egg-info/dependency_links.txt0000644000175100001660000000000115046547154020327 0ustar00runnerdocker ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975852.0 llm-0.27.1/llm.egg-info/entry_points.txt0000644000175100001660000000004415046547154017555 0ustar00runnerdocker[console_scripts] llm = llm.cli:cli ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975852.0 llm-0.27.1/llm.egg-info/requires.txt0000644000175100001660000000063515046547154016665 0ustar00runnerdockerclick condense-json>=0.1.3 openai>=1.55.3 click-default-group>=1.2.3 sqlite-utils>=3.37 sqlite-migrate>=0.1a2 pydantic>=2.0.0 PyYAML pluggy python-ulid setuptools pip puremagic [:sys_platform == "win32"] pyreadline3 [test] build click<8.2.0 pytest numpy pytest-httpx>=0.33.0 pytest-asyncio cogapp mypy>=1.10.0 black>=25.1.0 pytest-recording ruff syrupy types-click types-PyYAML types-setuptools llm-echo==0.3a3 ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975852.0 llm-0.27.1/llm.egg-info/top_level.txt0000644000175100001660000000000415046547154017005 0ustar00runnerdockerllm ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1754975845.0 llm-0.27.1/pyproject.toml0000644000175100001660000000404015046547145014715 0ustar00runnerdocker[project] name = "llm" version = "0.27.1" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "Simon Willison" }, ] license = "Apache-2.0" requires-python = ">=3.9" classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: End Users/Desktop", "Intended Audience :: Science/Research", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Text Processing :: Linguistic", "Topic :: Utilities", ] dependencies = [ "click", "condense-json>=0.1.3", "openai>=1.55.3", "click-default-group>=1.2.3", "sqlite-utils>=3.37", "sqlite-migrate>=0.1a2", "pydantic>=2.0.0", "PyYAML", "pluggy", "python-ulid", "setuptools", "pip", "pyreadline3; sys_platform == 'win32'", "puremagic", ] [project.urls] Homepage = "https://github.com/simonw/llm" Documentation = "https://llm.datasette.io/" Issues = "https://github.com/simonw/llm/issues" CI = "https://github.com/simonw/llm/actions" Changelog = "https://github.com/simonw/llm/releases" [project.scripts] llm = "llm.cli:cli" [project.optional-dependencies] test = [ "build", "click<8.2.0", # https://github.com/simonw/llm/issues/1024 "pytest", "numpy", "pytest-httpx>=0.33.0", "pytest-asyncio", "cogapp", "mypy>=1.10.0", "black>=25.1.0", "pytest-recording", "ruff", "syrupy", "types-click", "types-PyYAML", "types-setuptools", "llm-echo==0.3a3", ] [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1754975852.913846 llm-0.27.1/setup.cfg0000644000175100001660000000004615046547155013625 0ustar00runnerdocker[egg_info] tag_build = tag_date = 0