pax_global_header 0000666 0000000 0000000 00000000064 15257614025 0014521 g ustar 00root root 0000000 0000000 52 comment=b16bc062e5b92867a086cb6cc82c3b666a6bc4f9
cogent3-scinexus-b16bc06/ 0000775 0000000 0000000 00000000000 15257614025 0015327 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/.claude-plugin/ 0000775 0000000 0000000 00000000000 15257614025 0020136 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/.claude-plugin/marketplace.json 0000664 0000000 0000000 00000001225 15257614025 0023321 0 ustar 00root root 0000000 0000000 {
"name": "scinexus",
"owner": {
"name": "The scinexus authors",
"url": "https://github.com/cogent3/scinexus"
},
"description": "The scinexus agent skill, for building applications with scinexus",
"plugins": [
{
"name": "scinexus",
"source": "./",
"description": "Writing composable, type-checked data pipelines with scinexus: define_app, NotCompleted, data stores, apply_to, parallel backends",
"homepage": "https://scinexus.readthedocs.io",
"repository": "https://github.com/cogent3/scinexus",
"license": "BSD-3-Clause",
"keywords": ["scinexus", "pipelines", "scientific-computing"]
}
]
}
cogent3-scinexus-b16bc06/.claude-plugin/plugin.json 0000664 0000000 0000000 00000001106 15257614025 0022325 0 ustar 00root root 0000000 0000000 {
"name": "scinexus",
"displayName": "scinexus",
"version": "2026.10.2",
"description": "Writing composable, type-checked data pipelines with scinexus: define_app, NotCompleted, data stores, apply_to, parallel backends",
"author": {
"name": "The scinexus authors",
"url": "https://github.com/cogent3/scinexus"
},
"homepage": "https://scinexus.readthedocs.io",
"repository": "https://github.com/cogent3/scinexus",
"license": "BSD-3-Clause",
"keywords": [
"scinexus",
"pipelines",
"scientific-computing"
],
"skills": [
"./skills/"
]
} cogent3-scinexus-b16bc06/.github/ 0000775 0000000 0000000 00000000000 15257614025 0016667 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/.github/dependabot.yml 0000664 0000000 0000000 00000000722 15257614025 0021520 0 ustar 00root root 0000000 0000000 version: 2
updates:
- package-ecosystem: pip
directory: "/"
schedule:
interval: weekly
time: "19:00"
open-pull-requests-limit: 10
cooldown:
default-days: 7
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
time: "19:00"
open-pull-requests-limit: 10
cooldown:
default-days: 7
groups:
codeql-action:
patterns:
- "github/codeql-action*"
cogent3-scinexus-b16bc06/.github/workflows/ 0000775 0000000 0000000 00000000000 15257614025 0020724 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/.github/workflows/ci.yml 0000664 0000000 0000000 00000007157 15257614025 0022054 0 ustar 00root root 0000000 0000000 name: CI
on:
push:
pull_request:
# NOTE:
# if changing python versions, also update versions in
# - release.yml
# - noxfile.py
jobs:
tests:
name: "Python ${{ matrix.python-version }} (${{ matrix.os }})"
runs-on: ${{ matrix.os }}
strategy:
# without this a failure on an exploratory free-threaded build cancels
# the supported cells, which then never upload coverage
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ["3.11", "3.14", "3.14t"]
steps:
- uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1
with:
fetch-depth: 0
# Setup env
- uses: "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97" # v7.0.0
with:
python-version: "${{ matrix.python-version }}"
# nox is invoked with --force-python python, so it takes whatever is on
# PATH. A GIL-enabled build there would pass silently as a duplicate of
# the 3.14 cell rather than testing anything new.
- name: "Confirm the interpreter is free-threaded"
if: endsWith(matrix.python-version, 't')
shell: bash
run: python -c "import sys; print(sys.version); assert not sys._is_gil_enabled()"
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
cache-dependency-glob: "pyproject.toml"
- name: Install MPI (Ubuntu)
if: startsWith(matrix.os, 'ubuntu')
run: |
sudo apt-get update
sudo apt-get install -y openmpi-bin libopenmpi-dev
- name: "Run nox for ${{ matrix.python-version }}"
shell: bash
run: |
lname="snx-${{matrix.os}}-${{matrix.python-version}}.lcov"
xname="snx-${{matrix.os}}-${{matrix.python-version}}.xml"
cov="lcov -o$lname xml -o$xname"
uv run --group dev nox --force-python python -s testcov -- $cov
- name: Coveralls Parallel
uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8
env:
# macOS installs the coverage-reporter via a third-party Homebrew
# tap; Homebrew's new tap-trust enforcement refuses it otherwise.
HOMEBREW_NO_REQUIRE_TAP_TRUST: 1
with:
parallel: true
github-token: ${{ secrets.github_token }}
flag-name: run-${{matrix.python-version}}-${{matrix.os}}
file: "snx-${{matrix.os}}-${{matrix.python-version}}.lcov"
type_check:
name: Type Check
runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version: ["3.14"]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- uses: "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97" # v7.0.0
with:
python-version: "${{ matrix.python-version }}"
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
cache-dependency-glob: "pyproject.toml"
- name: "Run Type Checking for ${{ matrix.python-version }}"
run: |
uv run --group dev nox --force-python python -s type_check
finish:
name: "Finish Coveralls"
needs: tests
runs-on: ubuntu-latest
steps:
- name: Coveralls Finished
uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8
with:
github-token: ${{ secrets.github_token }}
parallel-finished: true cogent3-scinexus-b16bc06/.github/workflows/codeql.yml 0000664 0000000 0000000 00000001741 15257614025 0022721 0 ustar 00root root 0000000 0000000 name: "CodeQL"
on:
push:
branches-ignore:
- master
pull_request:
branches-ignore:
- master
schedule:
- cron: '39 20 * * 6'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'python' ]
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Initialize CodeQL
uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
with:
languages: ${{ matrix.language }}
- name: Autobuild
uses: github/codeql-action/autobuild@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
with:
category: "/language:${{matrix.language}}"
cogent3-scinexus-b16bc06/.github/workflows/docs.yml 0000664 0000000 0000000 00000004564 15257614025 0022410 0 ustar 00root root 0000000 0000000 name: Build Docs
concurrency:
group: docs-build-${{ github.ref }}
cancel-in-progress: true
on:
workflow_dispatch:
release:
types: [published]
push:
branches:
- main
paths:
- 'docs/**'
- 'overrides/**'
- 'skills/**'
- 'scripts/publish_skills.py'
- '.readthedocs.yaml'
- 'zensical.toml'
- 'rtd_get_docs.py'
permissions:
contents: read
jobs:
build-docs:
if: github.repository == 'cogent3/SciNexus'
runs-on: ubuntu-latest
steps:
- name: Checkout Repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
submodules: recursive
- uses: "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97" # v7.0.0
with:
python-version: "3.14"
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
cache-dependency-glob: "pyproject.toml"
- name: Build Documentation
run: |
# update executable components
uv run --group dev nox -s cogdocs
# build actual docs
uv run --group dev --group doc zensical build --clean
# agent skills, which zensical cannot place: it does not copy a
# hidden directory out of docs/
uv run python scripts/publish_skills.py site
working-directory: ${{ github.workspace }}
- name: Upload Documentation Artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: scinexus-docs-html
path: site
# without this the .well-known directory is dropped from the
# artifact and every published skill URL becomes a 404
include-hidden-files: true
trigger_rtd:
if: github.repository == 'cogent3/SciNexus'
runs-on: ubuntu-latest
needs: build-docs
steps:
- name: Trigger Read the Docs build
env:
RTDS_WEBHOOK_URL: ${{ secrets.RTDS_WEBHOOK_URL }}
RTDS_WEBHOOK_TOKEN: ${{ secrets.RTDS_WEBHOOK_TOKEN }}
BRANCH: ${{ github.ref_name }}
run: |
curl -sS --fail-with-body -X POST \
--data-urlencode "token=${RTDS_WEBHOOK_TOKEN}" \
--data-urlencode "branches=${BRANCH}" \
"${RTDS_WEBHOOK_URL}"
cogent3-scinexus-b16bc06/.github/workflows/linters.yml 0000664 0000000 0000000 00000001752 15257614025 0023134 0 ustar 00root root 0000000 0000000 name: Lint code using ruff
on:
push:
pull_request:
jobs:
linters:
runs-on: ubuntu-latest
if: github.repository != 'cogent3/SciNexus'
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.14'
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
cache-dependency-glob: "pyproject.toml"
- name: Format code using ruff
run: uv run --group dev nox -s fmt
- name: Commit changes
uses: EndBug/add-and-commit@cc9c08ba6c8df3b93a8f2db63e89b98368ae2ae8 # v11.1.1
with:
author_name: ${{ github.actor }}
author_email: ${{ github.actor }}@users.noreply.github.com
message: "STY: pre-commit linting with ruff"
add: "."
cogent3-scinexus-b16bc06/.github/workflows/release.yml 0000664 0000000 0000000 00000011552 15257614025 0023073 0 ustar 00root root 0000000 0000000 name: Release
on: [workflow_dispatch]
jobs:
test:
name: "Test on Python ${{ matrix.python-version }} (${{ matrix.os }})"
runs-on: ${{ matrix.os }}
strategy:
# without this a failure on an exploratory free-threaded build cancels
# the supported cells, hiding whether they would have passed
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ["3.11", "3.12", "3.13", "3.14", "3.14t"]
steps:
- uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1
with:
fetch-depth: 0
# Setup env
- uses: "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97" # v7.0.0
with:
python-version: "${{ matrix.python-version }}"
# nox is invoked with --force-python python, so it takes whatever is on
# PATH. A GIL-enabled build there would pass silently as a duplicate of
# the 3.14 cell rather than testing anything new.
- name: "Confirm the interpreter is free-threaded"
if: endsWith(matrix.python-version, 't')
shell: bash
run: python -c "import sys; print(sys.version); assert not sys._is_gil_enabled()"
- name: Install MPI (Ubuntu)
if: startsWith(matrix.os, 'ubuntu')
run: |
sudo apt-get update
sudo apt-get install -y openmpi-bin libopenmpi-dev
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
cache-dependency-glob: "pyproject.toml"
- name: "Run nox for ${{ matrix.python-version }}"
shell: bash
run: |
uv run --group dev nox -db uv --force-python python -s test
uv run --group dev nox -db uv --force-python python -s test_types
# the doc group pins cogent3[extra], which pulls kaleido and so orjson,
# and orjson publishes no free-threaded wheels
- name: "Run doc tests for ${{ matrix.python-version }}"
if: ${{ !endsWith(matrix.python-version, 't') }}
shell: bash
run: |
uv run --group dev nox -db uv --force-python python -s test_docs
- name: "Run MPI tests"
if: startsWith(matrix.os, 'ubuntu')
shell: bash
run: |
uv run --group dev nox -db uv --force-python python -s testmpi
docbuild:
name: "Test the docs"
runs-on: ubuntu-latest
steps:
- uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" # v7.0.1
with:
fetch-depth: 0
- uses: "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97" # v7.0.0
with:
python-version: "3.14"
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
cache-dependency-glob: "pyproject.toml"
- name: "test the docs code"
run: |
uv run --group dev nox -s cogdocs
uv run --group dev --group doc zensical build
build:
name: Build wheel and sdist
needs: test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.14'
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: true
cache-dependency-glob: "pyproject.toml"
- name: Build sdist and wheel
run: uv build
- name: Upload sdist and wheel
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: snx-wheel-sdist
path: |
./dist/*.whl
./dist/*.tar.gz
release_test:
name: Release to Test PyPI
needs: [build, docbuild]
environment: release_test
runs-on: ubuntu-latest
permissions:
id-token: write
steps:
- name: Download sdist and wheel
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: snx-wheel-sdist
path: ./dist
- name: Publish package distributions to Test PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
repository-url: https://test.pypi.org/legacy/
release:
name: Release to PyPI
needs: release_test
environment: release
runs-on: ubuntu-latest
permissions:
id-token: write
steps:
- name: Download sdist and wheel
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: snx-wheel-sdist
path: ./dist
- name: Publish package distributions to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
cogent3-scinexus-b16bc06/.gitignore 0000664 0000000 0000000 00000001152 15257614025 0017316 0 ustar 00root root 0000000 0000000 *.py[cod]
# C extensions
*.so
# Packages
*.egg
*.egg-info
dist
build
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64
# Installer logs
pip-log.txt
# Unit test / coverage reports
.coverage
.tox
.nox
*.nox
nosetests.xml
tests/draw_results
.pytest_cache
*.pytest_cache
.mypy_cache
tests/junit-*.xml
tests/coverage.xml
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
*.vscode
# vi
.*.swp
#Ignore vscode AI rules
.github/instructions/codacy.instructions.md
CLAUDE.md
.idea/*
.DS_Store
*.wpr
*.wpu
.cache/*
.claude/settings.local.json
__pycache__
_build
.codacy
working
site cogent3-scinexus-b16bc06/.hgignore 0000664 0000000 0000000 00000000627 15257614025 0017137 0 ustar 00root root 0000000 0000000 syntax:glob
.svn
*.pyc
*.pyo
*.so
*.o
*.DS_Store
*.tmproj
*.rej
*.orig
*.wpr
*.pdf
_build/*
build
*htmlcov*
*.idea
*.coverage*
*egg-info*
*.wpu
.cache*
*taskpaper
*.ipynb
*.ipynb_checkpoints*
*.sublime*
*.patch
*.pytest_cache
*.tox
*.nox
*.vscode
*.code-workspace
coverage.xml
__pycache__
junit-*.xml
dist/*
working/*
lcov*.info
.ruff_cache/*
.devcontainer/*
venv*
.venv*
.mypy*
CLAUDE.md
site/*
docs/data/* cogent3-scinexus-b16bc06/.readthedocs.yaml 0000664 0000000 0000000 00000001175 15257614025 0020562 0 ustar 00root root 0000000 0000000 # .readthedocs.yaml
# Read the Docs configuration file for MkDocs projects
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
# Required
version: 2
# Set the version of Python and other tools you might need
build:
os: ubuntu-22.04
tools:
python: "3.14"
commands:
# Install the required dependencies
- pip install requests
# Run the script to download and extract the pre-built docs
- python rtd_get_docs.py
- echo "Documentation downloaded and extracted"
# Disable the default build processes since we're using pre-built docs
sphinx:
configuration: null
python:
install: []
cogent3-scinexus-b16bc06/LICENSE 0000664 0000000 0000000 00000002726 15257614025 0016343 0 ustar 00root root 0000000 0000000 BSD 3-Clause License
Copyright (c) 2026, cogent3
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cogent3-scinexus-b16bc06/README.md 0000664 0000000 0000000 00000013421 15257614025 0016607 0 ustar 00root root 0000000 0000000
[](https://coveralls.io/github/cogent3/scinexus?branch=main) [](https://app.codacy.com/gh/cogent3/scinexus/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) [](https://github.com/astral-sh/ruff) [](https://github.com/cogent3/scinexus/actions/workflows/ci.yml) 
*`scinexus` is a framework for rapid development of data processing applications. It enables interoperability between objects through defined data types, allowing development of scientific domain app ecosystems. Just as `attrs` and `dataclasses` use type hints to simplify data type definition, `scinexus` uses them to simplify writing best-practice scientific algorithms.*
Many scientific problems require repeating calculations across many files or database records. Such tasks suit data-level parallelism, but writing robust, maintainable code for them is often tedious and quickly becomes complex.
As the Unix philosophy articulates, writing algorithms that do one thing well and can be composed together through piping data of known type is a *Very Good Thing*â„¢.
**`scinexus` encourages this design pattern and eliminates the boilerplate.** We leverage the Python type annotation system to govern the compatibility (composability) of different applications. This enables in-process composition of your applications with validation of the consistency of the pipeline and the consistency of the data being run through it.
**`scinexus` is designed for scientific reproducibility.** Scientific computations should record all conditions needed to reproduce an analysis. `scinexus` reduces the effort by intercepting all arguments (including defaults) used in app construction and logging the resulting app state.
## Examples
Developers can choose inheriting from a base class or use the `scinexus.define_app` decorator to make composable apps. The following examples show simple composition
Loading files so missing data does not cause a crash
```python
from scinexus import define_app
@define_app(app_type="loader")
def read_json(path: str) -> dict:
import json
with open(path) as f:
return json.load(f)
@define_app
def validate(data: dict, required_field: str) -> dict:
if required_field not in data:
# this becomes a NotCompleted sentinel object
# your run doesn't crash!
raise ValueError(f"missing {required_field!r} field")
return data
app = read_json() + validate(required_field="name")
```
You can apply `app` to a single file path as `app(filepath)`, or operate in parallel (and show a progress bar) on a sequence of file paths as
```python
results = list(app.as_completed(["some_file_path.json", "some_other_file_path.json"], parallel=True, show_progress=True)
```
A contrived numerical example
```python
from scinexus import define_app
@define_app
def normalise(values: list[float]) -> list[float]:
lo, hi = min(values), max(values)
return [(v - lo) / (hi - lo) for v in values]
@define_app
def threshold(values: list[float]) -> list[bool]:
return [v > 0.5 for v in values]
app = normalise() + threshold()
app([1.0, 5.0, 3.0, 9.0])
```
A configurable app
```python
from scinexus import define_app
@define_app(app_type="loader")
def load_csv(path: str) -> list[dict]:
import csv
with open(path) as f:
return list(csv.DictReader(f))
@define_app
class summarise:
def __init__(self, column: str) -> None:
"""column contains the values to produce summary stats for"""
self.column = column
def main(self, rows: list[dict]) -> dict[str, float]:
vals = [float(r[self.column]) for r in rows]
return {"mean": sum(vals) / len(vals), "min": min(vals), "max": max(vals)}
app = load_csv() + summarise(column="price")
```
## Features
- Type checking at composition time
- Durable computing -- failures recorded as `NotCompleted` records, not exceptions
- Data-level parallel execution with pluggable backends (stdlib, loky, MPI, or custom)
- Progress bars (`tqdm` or `rich`)
- Automated logging and citation tracking
- Checkpointing via data stores (directory, SQLite)
## Installation
```bash
pip install scinexus
```
## The `scinexus` origin story
The app framework and utility functions in `scinexus` incubated inside [cogent3](https://github.com/cogent3/cogent3) from March 2019, accumulating over seven years of development, testing, and real-world use in computational genomics before being extracted into a standalone package. The design is mature and has underpinned analyses in published studies.
The extraction into `scinexus` makes the infrastructure available to any scientific Python project, free of the `cogent3` dependency. See the [changelog](changelog.md) for a detailed list of changes from the cogent3 app infrastructure.
We acknowledge here that many members of the `cogent3` community contributed to the code that now lives here, including [@GavinHuttley](https://github.com/GavinHuttley), [@rmcar17](https://github.com/rmcar17), [@Nick-Foto](https://github.com/Nick-Foto), [@KatherineCaley](https://github.com/KatherineCaley), [@fredjaya](https://github.com/fredjaya), and [@khiron](https://github.com/khiron).
cogent3-scinexus-b16bc06/changelog.d/ 0000775 0000000 0000000 00000000000 15257614025 0017500 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/changelog.d/README.md 0000664 0000000 0000000 00000003633 15257614025 0020764 0 ustar 00root root 0000000 0000000 # Using `scriv`
## Adding changelog entries
This directory is used to store changelog fragments for the next release
of the project. Each file should contain a small changelog fragment
that will be added to the full changelog when the release is made.
The file is created using
```
scriv create --edit
```
to create a change entry
This will create a file with the correct name and format and commented out sample categories, and load it into your `git` editor (the one specified by ```git config --global core.editor``` )
Uncomment the category of change you are making and add a short description of the
change as a markdown bullet point. For example:
Contributors
* khiron
Enhancements
* Added a new feature that allows users to do Y
Deprecations
* Removed deprecated feature Z
Bug fixes
* Fixed a bug that caused the project to crash when a user did X by doing Y instead
Documentation
* Documented feature Z
Check the file in with your changes.
---
## Building a changelog
To build the changelog for the next release, run
```
scriv collect
```
This will create a file called `CHANGELOG.md` in the root of the project. This file will contain the full changelog for the project, including all the fragments that have been added to the changelog.d directory and remove those fragments. Note this requires that the `__init__.py` file's `__version__` variable is updated from the last time you ran collect, otherwise you will get a warning
```
Entry 'Changes since release 0.0.1' already uses version '0.0.1'.
```
... and the fragments will not be collected. When the `__version__` variable in the project is updated the next time you run collect, the fragments will be collected and the changelog will be updated.
You can override this to create a new collection of all available fragments that are not aligned with a `__version__` with
```
scriv collect --version "description of a milestone not yet used"
```
cogent3-scinexus-b16bc06/changelog.d/templates/ 0000775 0000000 0000000 00000000000 15257614025 0021476 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/changelog.d/templates/new.md.j2 0000664 0000000 0000000 00000000354 15257614025 0023125 0 ustar 00root root 0000000 0000000
{% for cat in config.categories -%}
{% endfor -%}
cogent3-scinexus-b16bc06/changelog.d/templates/title.md.j2 0000664 0000000 0000000 00000000040 15257614025 0023445 0 ustar 00root root 0000000 0000000 Changes in release "{{version}}" cogent3-scinexus-b16bc06/changelog.md 0000664 0000000 0000000 00000015625 15257614025 0017611 0 ustar 00root root 0000000 0000000 # Changelog
## Changes in release "2026.10.2"
### API
- A writer's own `id_from_source` now names the records it writes. Before,
`__call__`, `apply_to` and `as_completed` always used the global
`get_id_from_source()` and ignored the writer's own setting. A custom
extractor must be picklable under `parallel=True`, and `apply_to` uses it
to work out which records are already done.
### Contributors
- @GavinHuttley
### Enhancements
- Apps now record which constructor arguments were passed explicitly,
separately from the ones that took their defaults.
- When a SQLite store's lock is refused, the message now says who holds it:
this thread, another thread in this process, or another process. For the
first two, it suggests reusing or closing the open store instead of
`unlock(force=True)`.
### Bug fixes
- Member ids are spelled with `/` on every platform, including Windows. This
covers SQLite log ids too.
- Passing `id_from_source` to `as_completed` or `apply_to` no longer leaks
into later calls. Re-composing an applied writer no longer raises
`TypeError`.
- Calling `lock()` on a store that already holds its lock no longer fails.
### Documentation
- Added a vendor-neutral scinexus agent skill (`skills/scinexus/SKILL.md`)
with reference files on pipelines, IO and extending.
- The docs site now publishes the skill at the standard
`.well-known/agent-skills/` paths.
- Improved the wording of the parallel-usage docs and the developer docs.
## Differences from cogent3
Changes from the original cogent3 app infrastructure.
### New Features
- Standalone package extracted from `cogent3.app` — no cogent3 dependency required.
- Generic base classes `AppBase[T, R]`, `ComposableApp[T, R]`, and `WriterApp[T, R]` that apps can inherit from directly as an alternative to the `@define_app` decorator. Type checkers can resolve types through class inheritance without a plugin.
- mypy plugin (`scinexus._mypy_plugin`) for correct type inference of `@define_app` decorated classes. Synthesises the `__call__` return type as `R | NotCompleted`.
- `check_data_type` attribute on apps — a settable property to toggle runtime input type checking on or off. Disabling can speed up execution and simplify debugging.
- `NotCompletedType` enum (`ERROR`, `FAIL`, `BUG`) for categorising failure types, replacing bare strings.
- `set_summary_display()` / `get_summary_display()` — a module-level registry allowing downstream packages (e.g. cogent3) to register custom display functions for data store summary methods (`describe`, `summary_logs`, `summary_not_completed`, `summary_citations`, `validate`).
- `citations` and `bib` properties on apps for tracking software citations via the `citeable` library. Citations propagate through composed pipelines.
- Pluggable parallel backends -- choose between stdlib multiprocessing, loky, or MPI via `set_parallel_backend()`.
- Pluggable progress bars -- use `tqdm` or `rich` via the `Progress` protocol and `set_default_progress()`.
- `set_id_from_source()` / `get_id_from_source()` -- register a custom function for extracting storage identifiers from data.
- `apply_to()` accepts `logger=False` to disable log file creation.
### Enhancements
- App composition (`+`) now makes shallow copies of the right-hand operand. Composed pipelines no longer share mutable state.
- Composition-time type compatibility checking via `check_type_compatibility()` — catches type mismatches when apps are composed with `+`, before any data is processed.
- Data store summary methods (`describe`, `summary_logs`, etc.) return `list[dict]` or `dict` instead of cogent3 `Table` objects. Custom display can be restored via `set_summary_display()`.
- All modules pass mypy strict type checking.
- `StrOrBytes` type alias replaced with `str | bytes` throughout.
- Inline `assert` statements replaced with explicit `ValueError` / `TypeError` raises.
- Type-hint-related imports moved under `TYPE_CHECKING` for lighter runtime import overhead.
- `max_workers` below 1 is refused by the local backends with a message naming the value, where it previously reached the executor and surfaced that library's own error. `max_workers=0` was a silent synonym for `None` and is now an error, and a `bool` raises `TypeError` where `True` previously asked for one worker, so `None` is the only way to ask for one worker per CPU. The MPI backend is unchanged and still reads `0` as one worker.
- `chunksize` follows the same rule as `max_workers`: below 1 raises `ValueError` and a `bool` raises `TypeError`, where `0` and `False` previously meant "no preference" and `True` asked for one item per chunk. Every call that accepts the argument now checks it, including the ones that go on to ignore it — only `imap` on the process and MPI backends chunks the work by it, since `as_completed` submits one task per item and a thread pool has no per-item transport cost to amortise.
- A writable `DataStoreSqlite` is created only by the master process, matching `DataStoreDirectory`, which has always returned early off the master. Construction off the master still succeeds, and the first read or write that reaches a database which does not exist, or a file carrying no schema, now raises `OSError` instead of creating the database and its tables. So an app that opens a writable store of its own inside a worker no longer works, as it already did not for a directory store. Using a store the master made is unchanged, and an in-memory store is exempt: it has no file to race over, so refusing would remove it outright for no gain.
- `open_()` takes a bare `"r"` or `"w"` as text for every suffix, matching builtin `open` rather than `gzip`, `bz2` and `lzma`, which take a bare mode as binary. Previously `open_(path, "w")` on a `.gz` gave a binary handle and `open_(path, "r")` raised. Use `"rb"` or `"wb"` for bytes. This also applies to `atomic_write`, whose mode defaults to `"w"`.
- A writer constructed with an `id_from_source` argument now names records with it. Calling the app, `apply_to()` and `as_completed()` all consult it before the function registered through `set_id_from_source()`, so a writer that takes its own callable is no longer overruled by the framework. Passing `id_from_source=` to `apply_to()` or `as_completed()`, or `identifier=` to the call, still wins over both. Only a writer is consulted, since `id_from_source` means whatever its author chose on any other app type. The user provided function must be picklable to run under `parallel=True`.
- `id_from_source` passed to `apply_to()` or `as_completed()` is a setting on the app rather than a one-off. It applies to later calls of either method until another is given, and on a writer to calling the app as well, and it outranks the `id_from_source` a writer was constructed with.
### Deprecated
- `ComposableApp.disconnect()` — discontinued, will be removed in version 2026.9. No longer required since composition uses shallow copies.
cogent3-scinexus-b16bc06/docs/ 0000775 0000000 0000000 00000000000 15257614025 0016257 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/docs/conftest.py 0000664 0000000 0000000 00000000314 15257614025 0020454 0 ustar 00root root 0000000 0000000 from scinexus.typing import register_type_namespace
try:
from cogent3.app.typing import _get_resolution_namespace
register_type_namespace(_get_resolution_namespace)
except ImportError:
pass
cogent3-scinexus-b16bc06/docs/explanation/ 0000775 0000000 0000000 00000000000 15257614025 0020601 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/docs/explanation/app-lifecycle.md 0000664 0000000 0000000 00000002202 15257614025 0023634 0 ustar 00root root 0000000 0000000 # The app lifecycle
!!! abstract ""
The different app types, their base classes, and how `define_app` transforms a user defined class or function into a composable app.
## Types of apps
### Loaders
These are responsible for loading data and are composable. They inherit from `LoaderApp`.
```python
from scinexus import LoaderApp
```
### Writers
These are responsible for writing data and are composable. They inherit from `WriterApp`.
```python
from scinexus import WriterApp
```
### Generic
Generic apps do other operations on data and are composable. They inherit from `ComposableApp`
```python
from scinexus import ComposableApp
```
### Non-composable
Non-composable apps cannot be combined with other apps into pipelines.
```python
from scinexus import NonComposableApp
```
!!! info
You can create your app by inheriting from one of the above base classes. Or you can use the `define_app` decorator. Using the decorator is the fastest way to turn something you already have into a composable app. Under the hood, the decorator is basically injecting the base classes described above into the inheritance of your own classes.
cogent3-scinexus-b16bc06/docs/explanation/customisation-hooks.md 0000664 0000000 0000000 00000010151 15257614025 0025143 0 ustar 00root root 0000000 0000000 # Customisation hooks
!!! abstract ""
How `scinexus` uses module-level registry functions to let downstream packages customise summary display and identifier extraction without subclassing.
## The pattern
`scinexus` uses module-level registry functions to let downstream packages customise behaviour without subclassing or monkey-patching. Each hook follows the same shape:
- **`set_*(func)`** registers a callable (or `None` to clear)
- **`get_*()`** returns the currently registered callable (or the default)
This keeps `scinexus` free of dependencies on downstream packages while still allowing them to integrate deeply.
## `set_summary_display` — transforming summary output
Data store summary properties (`.describe`, `.summary_logs`, `.summary_not_completed`, `.summary_citations`, `.validate()`) collect their data as plain Python dicts or lists of dicts. By default these are returned as-is.
A downstream package can register a display function that transforms these raw structures into richer objects. The function must accept `(data, *, name)` where `data` is the raw dict or list and `name` is the summary method name (e.g. `"describe"`).
### How `cogent3` uses this
When `cogent3.app` is imported, it registers a function that converts summaries into `cogent3.core.table.Table` objects:
```python { notest }
from scinexus.data_store import set_summary_display
from cogent3.core.table import Table
def _summary_to_table(data, *, name):
if isinstance(data, dict):
title = data.pop("title", name)
rows = [[k, v] for k, v in data.items()]
return Table(
header=["Condition", "Value"],
data=rows,
title=title,
)
if isinstance(data, list):
if not data:
return Table(header=[], data=[], title=name)
header = list(data[0].keys())
rows = [list(row.values()) for row in data]
return Table(header=header, data=rows, title=name)
return data
set_summary_display(_summary_to_table)
```
After this registration, every call to `dstore.describe` or `dstore.summary_not_completed` returns a `Table` with a rich notebook repr, rather than a plain dict.
## `set_id_from_source` — customising unique ID extraction
When `apply_to()` or `as_completed()` processes a data store, each result needs a unique identifier so the writer can store it and skip already-processed inputs on subsequent runs. By default, `scinexus` extracts this ID using `get_unique_id`, which strips format suffixes from file names:
```
"gene_001.fasta.gz" → "gene_001"
```
If your data uses a different naming convention — for example, IDs embedded in the file content or in a metadata field — you can register a custom extractor:
```python { notest }
from scinexus.data_store import set_id_from_source
def my_id_extractor(data):
"""Extract ID from a metadata dict."""
return data.info.source.split("/")[-1].split("_")[0]
set_id_from_source(my_id_extractor)
```
The registered function is consulted by:
- `WriterApp.apply_to()` — to derive output record keys
- `AppBase.as_completed()` — to identify results
- `NotCompleted` — to normalise the `source=` attribute on error records
Pass `None` to restore the default:
```python { notest }
set_id_from_source(None) # back to get_unique_id
```
Per-call overrides via the `id_from_source` keyword on `apply_to()` and `as_completed()` still take precedence over the registered function.
## The default ID pipeline: `get_data_source` → `get_unique_id`
The default extractor, `get_unique_id`, works in two steps:
1. **`get_data_source(data)`** extracts a source string from the input. This is a singledispatch function that handles:
- `str` / `Path` → the file name
- `dict` → looks for `data["info"]["source"]` or `data["source"]`
- `DataMemberABC` → the member's `unique_id`
- Any object with a `.source` attribute → recurses on that attribute
2. **`get_unique_id(name)`** strips format suffixes (e.g. `.fasta`, `.gz`) from the source string returned by `get_data_source`.
Together they turn inputs like `DataMember(unique_id="gene_001.fasta.gz")` into the key `"gene_001"`.
cogent3-scinexus-b16bc06/docs/explanation/data-store-model.md 0000664 0000000 0000000 00000006232 15257614025 0024267 0 ustar 00root root 0000000 0000000 # Data store model
!!! abstract ""
How data stores map inputs to outputs via unique IDs, how checkpointing works to skip already-processed items, the three backends (directory, zip, SQLite) and when to use each, and how citations and logs are stored alongside results.
## Data stores -- collections of data records
If you download [raw.zip](../data/raw.zip) and unzip it, you will see it contains 1,035 files ending with a `.fa` filename suffix. (It also contains a tab delimited file and a log file, which we ignore for now.) The directory `raw` is a "data store" and the `.fa` files are "members" of it. In summary, a data store is a collection of members of the same "type". This means we can apply the same application to every member.
### Types of data store
| Class Name | Supported Operations | Supported Data Types | Identifying Suffix |
|---|---|---|---|
| `DataStoreDirectory` | read / write / append | text | None |
| `ReadOnlyDataStoreZipped` | read | text | `.zip` |
| `DataStoreSqlite` | read, write, append | text or bytes | `.sqlitedb` |
!!! note
The `ReadOnlyDataStoreZipped` is just a compressed `DataStoreDirectory`.
### The structure of data stores
If a directory was not created by `scinexus` as a `DataStoreDirectory` then it has only the structure that existed previously.
If a data store was created by `scinexus`, either as a directory or as a `sqlitedb`, then it contains four types of data: completed records, *not* completed records, log files and md5 files. In a `DataStoreDirectory`, these are organised using the file system. The completed members are valid data records (as distinct from not completed) and are at the top level. The remaining types are in subdirectories.
```
demo_dstore
├── logs
├── md5
├── not_completed
└── ...
```
`logs/` stores `scitrack` log files produced by `scinexus` writer apps. `md5/` stores plain text files with the md5 sum of a corresponding data member which are used to check the integrity of the data store. A checksum file is named for the record it belongs to and for which kind of record that is: `md5/id_0.cmplt` for a completed record and `md5/id_0.ncmplt` for a not completed one.
!!! note
Earlier versions kept both kinds under `md5/id_0.txt`, and reading such a store still works, since `md5()` falls back to that name. `validate()` counts them under `md5_legacy`, and `migrate_checksums()` renames the ones it can attribute:
```python { notest }
dstore = open_data_store("results", suffix="fasta", mode="w")
dstore.migrate_checksums()
# {'migrated': 12, 'ambiguous': ['id_3'], 'orphaned': ['id_9'], 'superseded': []}
```
`migrate_checksums()` needs `mode="w"`, because it rewrites files already in the store. A file is reported rather than renamed in three cases: `ambiguous` when both kinds of record carry its name, since it holds whichever of them wrote last and that was never recorded; `orphaned` when no record carries it; and `superseded` when that record already has a checksum under the current name, which was written for it by this version and so is the one to trust.
The `DataStoreSqlite` stores the same information, just in SQL tables.
cogent3-scinexus-b16bc06/docs/explanation/flow.md 0000664 0000000 0000000 00000004365 15257614025 0022102 0 ustar 00root root 0000000 0000000 # Execution flow of a composed app
!!! abstract ""
How data flows through a composed pipeline, step by step.
Consider two apps composed into a pipeline:
```python { notest }
from scinexus import define_app
@define_app(app_type="loader")
def read_json(path: str) -> dict:
import json
with open(path) as f:
return json.load(f)
@define_app
def validate(data: dict, required_field: str) -> dict:
if required_field not in data:
raise ValueError(f"missing {required_field!r} field")
return data
app = read_json() + validate(required_field="name")
```
Composing with `+` creates a new app where `validate` is the outermost app and `read_json` is stored as its `.input` attribute. When you call `app(filepath)`, execution begins at the outermost app and works inward.
## The execution flow when you call `app(filepath)`
```mermaid
flowchart TD
entry["Executes scinexus __call__(val)"] --> none{val is None?}
none -- yes --> nc_none[create and return NotCompleted ERROR, recording current app as origin]
none -- no --> nc{val is NotCompleted?}
nc -- yes --> nc_return[returns same NotCompleted]
nc -- no --> has_input{has an input app?}
has_input -- yes --> call_input["call input(val), which enters the top of this chart"]
call_input --> input_nc{result is NotCompleted?}
input_nc -- yes --> nc_input[return same NotCompleted]
input_nc -- no --> type_check
has_input -- no --> type_check{val type is valid for self}
type_check -- fail --> nc_type[create and return NotCompleted ERROR]
type_check -- pass --> main["main(val)"]
main -- exception --> nc_main[NotCompleted ERROR]
main -- success --> result["return result (which may be NotCompleted FAIL)"]
classDef errorNode fill:#fde0c8,stroke:#333
classDef successNode fill:#c8e0fd,stroke:#333
class nc_none,nc_return,nc_input,nc_type,nc_main errorNode
class result successNode
```
This is the same sequence for every composed app, regardless of pipeline length. Each app in the chain runs the same `__call__` checks, so `NotCompleted` propagation and exception handling are consistent throughout. See [Runtime type checking](type-system.md#runtime-type-checking) for details on how type validation works and how to disable it.
cogent3-scinexus-b16bc06/docs/explanation/index.md 0000664 0000000 0000000 00000001471 15257614025 0022235 0 ustar 00root root 0000000 0000000 # Explanation
Background and design rationale behind `scinexus`.
- [Why composable apps?](why-composable-apps.md) -- the problem `scinexus` solves and how it compares to alternatives
- [The app lifecycle](app-lifecycle.md) -- base classes and `define_app`
- [Type system](type-system.md) -- how composition-time type checking works
- [Execution flow](flow.md) -- step-by-step data flow through a composed pipeline
- [NotCompleted design](not-completed-design.md) -- why a sentinel pattern instead of exceptions
- [Source tracking](source-tracking.md) -- how `source_proxy` tracks data provenance through pipelines
- [Customisation hooks](customisation-hooks.md) -- `set_summary_display` and `set_id_from_source` registry functions
- [Data store model](data-store-model.md) -- unique IDs, checkpointing, and backend choices
cogent3-scinexus-b16bc06/docs/explanation/mpi-backend.md 0000664 0000000 0000000 00000011562 15257614025 0023302 0 ustar 00root root 0000000 0000000 # How the MPI backend differs
!!! abstract ""
Why MPI is not simply a fourth process pool: the worker count is decided outside Python by `mpiexec`, rank 0 is the master and does no work, `max_workers` cannot change any of it, and `UNIVERSE_SIZE` counts slots rather than ranks.
The `"multiprocess"`, `"threads"` and `"loky"` backends all assume the running process decides how much parallelism to use by looking at the machine it is on. `max_workers=4` means four workers, and leaving it out means one per CPU.
MPI breaks that assumption. The processes were created before Python started, by `mpiexec` or by a cluster scheduler, and nothing inside the program can add to them or give them back. Almost everything else on this page follows from that.
## The launcher decides
With a local backend the chain is short. You ask for workers, and the pool starts them:
```python { notest }
from scinexus import parallel
results = parallel.map(process, items, max_workers=4) # four workers
```
Under MPI that same argument is a request about an allocation that already exists and cannot be changed. The number that decides your parallelism is the one you gave the launcher:
```bash
mpiexec -n 8 python -m mpi4py.futures my_script.py
```
So under MPI, `mpiexec -n` is the single place the worker count is in Python set so set `use_mpi=True` but leave `max_workers` out.
## Rank 0 is the master and does no work
Every MPI process is a *rank*. Rank 0 is the master: it holds the data, hands out tasks and collects results, and does not process items itself. A job of 8 ranks therefore has **7 workers**, not 8.
This has no equivalent in the local backends, where a pool of 4 workers on a 4 core machine puts all 4 to work while the parent simply waits. Setting `max_workers=$PBS_NCPUS` on a cluster is wrong because it asks for one more worker than the job can ever contain.
It also means `-n 2` leaves you with a single worker, which is serial execution but using two processes. With `if_serial` defaulting to `"raise"`, `scinexus` will kill a job with one worker You can override this by setting `if_serial="ignore"`.
!!! info
`-n 3` is the smallest launch with more than one worker.
## Why the ranks must exist before the program starts
`scinexus` drives every backend through `concurrent.futures`, and for MPI that means you MUST launch your script using:
```bash
mpiexec -n 4 python -m mpi4py.futures my_script.py
```
!!! warning
`scinexus` is not written for the any other pattern, such as using `mpi4py` itself to spawn workers on demand while the program runs.
## `max_workers` cannot change the worker count
Because the pool is fixed before your code runs, `max_workers` has no way to grow or shrink it.
`scinexus` therefore treats the argument as a statement about an allocation rather than a request for one:
- **Left out**, you get the pool the job was launched with. This is the right choice almost always.
- **Equal to the pool size**, it is accepted quietly, since it agrees with reality.
- **Anything else**, you get a warning saying so, in both directions. Asking for fewer used to run on the whole pool in silence, which is the more surprising of the two.
- **Below one, or not a whole number**, it is refused, exactly as the local backends refuse it.
## `UNIVERSE_SIZE` counts slots, not ranks
MPI exposes an attribute called `UNIVERSE_SIZE`, which is easy to mistake for the number of processes. It is the number of *slots* the job may occupy, which usually means cores on the machine or slots granted by the scheduler.
The two numbers are independent, and neither is the worker count. Measured on a six core machine:
| launch | `UNIVERSE_SIZE` | ranks | workers |
|---|---|---|---|
| `mpiexec -n 4 -m mpi4py.futures` | 6 | 4 | 3 |
| `mpiexec --oversubscribe -n 8 -m mpi4py.futures` | 6 | 8 | 7 |
| `mpiexec -n 1 -m mpi4py.futures` | 6 | 1 | 1 |
Running more ranks than there are slots is **oversubscription**, which `mpiexec --oversubscribe` permits deliberately. It is a legitimate thing to do, for testing on a small machine or for work that spends its time waiting rather than computing. The third row is the other end, and it is a launch that will stop rather than run: one worker is serial, so `if_serial` refuses it by default.
!!! note
`get_size()` reports the last column, the workers you will actually get. It does not report `UNIVERSE_SIZE` and it does not report the rank count.
## What carries over unchanged
Two things behave as they do for the process backends rather than being MPI peculiarities. Arguments and results are pickled, so closures and lambdas are refused just as they are under `"multiprocess"`. And the main logic must sit behind an `if __name__ == "__main__":` guard, for the same reason it must under spawn based multiprocessing.
For the practical steps -- installing mpi4py, setting the backend, writing and launching the script -- see [Run in parallel](../howto/run-in-parallel.md).
cogent3-scinexus-b16bc06/docs/explanation/not-completed-design.md 0000664 0000000 0000000 00000011255 15257614025 0025150 0 ustar 00root root 0000000 0000000 # `NotCompleted` design
!!! abstract ""
Why `scinexus` uses a sentinel object instead of exceptions for handling failures in batch processing.
## The problem with exceptions in pipelines
When applying an algorithm to hundreds or thousands of data records, some records will inevitably fail — bad data, missing fields, violated preconditions. If failures raise exceptions, you face an unpleasant choice:
- **Let it crash.** You lose all progress and must restart from scratch.
- **Wrap everything in try/except.** Your pipeline logic becomes cluttered with error-handling boilerplate, and you must decide at every step what to catch and what to re-raise.
Neither approach scales well. You want failures to be recorded and the pipeline to continue processing the remaining records.
## The sentinel pattern
`NotCompleted` is `scinexus`'s answer: a sentinel return value that signals "this record could not be processed" without raising an exception. It carries structured information about the failure:
- **`.type`** — `FALSE` (a condition was not met) or `ERROR` (an unexpected exception occurred)
- **`.origin`** — which app produced the failure
- **`.source`** — which input data failed
- **`.message`** — a human-readable explanation
Because `NotCompleted` is a regular return value, it flows through the same code paths as successful results.
## Why it subclasses `int` and is falsy
`NotCompleted` subclasses `int` with a value of `0`, making it evaluate to `False` in boolean contexts. This means you can check for failure with a simple truthiness test:
```python { notest }
result = my_app(data)
if not result:
print(f"Failed: {result.message}")
```
Subclassing `int` rather than defining `__bool__` alone ensures consistent behaviour with Python's truth-testing protocol across all contexts (including NumPy arrays and other libraries that inspect types).
## Automatic propagation through pipelines
When apps are composed with `+`, the resulting pipeline checks each intermediate result. If any step returns a `NotCompleted`, subsequent steps are skipped and the `NotCompleted` is returned as the final result. This means:
- A single failure does not corrupt downstream steps.
- The failure's `.origin` accurately records where the problem occurred, not where it was finally caught.
- No try/except scaffolding is needed in pipeline code.
```python { linenums="1" notest }
import cogent3 as c3
aln = c3.get_dataset("primate-brca1")
select_seqs = c3.get_app("take_named_seqs", "Mouse", "Human")
min_length = c3.get_app("min_length", 300)
app = select_seqs + min_length
result = app(aln)
print(result)
# NotCompleted(type=FAIL, origin=take_named_seqs, source="brca1", message="named
# seq(s) {'Mouse'} not in ('FlyingLem', 'TreeShrew', 'Galago', 'HowlerMon',
# 'Rhesus', 'Orangutan', 'Gorilla', 'Chimpanzee', 'Human')")
```
## The one case that still raises
!!! warning "A writer that cannot name a record raises rather than returning `NotCompleted`"
Everything above describes failures of the *data*, and each one stays traceable to the record that caused it — that is what `.source` is for. A result that cannot be named breaks the guarantee rather than exercising it: there is nothing to attribute it to, so returning a `NotCompleted` would put an entry in the audit trail that no one can tie back to an input. `scinexus` raises `ValueError` instead, at the call that made the mistake.
This applies to failures too. An unattributable `NotCompleted` reaching a writer raises rather than propagating, for the same reason.
It happens only on a one-off call, `app(data)`, where neither the input nor the result carries an identity and no `identifier=` was passed. `apply_to()` always derives one from the data store, so it never hits this.
## Recording failures in data stores
When a pipeline is run via `apply_to()` on a data store, `NotCompleted` results are passed to the writer, which is expected to store them in a separate area (the `not_completed/` subdirectory or SQL table). This gives you a complete audit trail: you can inspect which records failed, which app was responsible, and why — all without interrupting the processing of successful records. A writer's `main()` always receives `NotCompleted` values, so it must handle them — see [Handle failures](../howto/handle-failures.md).
See [Handle failures](../howto/handle-failures.md) for usage examples.
cogent3-scinexus-b16bc06/docs/explanation/source-tracking.md 0000664 0000000 0000000 00000007562 15257614025 0024235 0 ustar 00root root 0000000 0000000 # Source tracking
!!! abstract ""
How `source_proxy` preserves the link between input identity and output when data is transformed through a pipeline.
## The problem
When you call `apply_to()` or `as_completed()` on a data store, each member is fed through the pipeline independently. The pipeline may transform the data into something completely different — a new object with no reference back to the input that produced it. But the writer at the end of the pipeline needs to know *which input* produced *which output* so it can assign the correct unique ID in the output data store.
For example, if a loader reads `"gene_001.fa"` and the pipeline returns a translated protein sequence, the writer needs to store that result under the key `"gene_001"`. Without a mechanism to carry the input identity forward, this link is lost.
## How `source_proxy` solves it
`source_proxy` is a transparent wrapper that carries two extra pieces of state alongside the wrapped object:
- **`.source`** — the original input (or its identifier), preserved across transformations
- **`.uuid`** — a unique identifier for this proxy instance, used for hashing
When `as_completed()` or `apply_to()` processes a data store, a member that does not have identifying information[^1] is wrapped in a `source_proxy` before entering the pipeline. Because `source_proxy` delegates attribute access to the wrapped object via `__getattr__`, downstream apps see the original object and do not need to know about the proxy.
```python { notest }
from scinexus.composable import source_proxy
proxy = source_proxy(some_data)
proxy.source # the original input
proxy.uuid # unique identifier for this proxy
proxy.any_attr # delegates to some_data.any_attr
```
## How `propagate_source` preserves the link
`propagate_source` wraps the whole composed pipeline rather than each step. A composed app pulls its input through the chain internally, so this runs once per input record, and when the pipeline returns the result is re-associated with the original source:
1. If the result carries a reference to its own origin[^1], the proxy is **unwrapped** — the result stands on its own.
2. Otherwise[^2] the proxy's wrapped object is **updated** to the new result via `set_obj()`, and the proxy, still carrying the original `.source`, is returned.
[^1]: a `.source` attribute for an object, or a `"source"` key, or `info["source"]`, for a `dict`
[^2]: For example, a `str` or `bytes` have no `.source` and take the second path.
This means the source identity survives an arbitrary number of pipeline steps, even when intermediate apps return entirely new objects.
## Why this matters for writers
`WriterApp.apply_to()` uses the source to derive unique IDs for output records. This enables **append-only semantics**: on a subsequent run against the same data store, records that already exist in the output are skipped. The unique ID comes from the original input's identity, extracted by whatever `get_id_from_source()` returns — by default `get_unique_id()`, which strips format suffixes, so an input `gene_001.fa` is stored under `gene_001`.
Without source tracking, the writer would have no way to determine whether a result corresponds to an input that has already been processed.
## Failures carry the same identity
When an app raises an exception, the framework turns it into a `NotCompleted` recording which input failed. That identity is taken on entry to the app, before the pipeline's intermediate result replaces it, so a failure part way down a chain still names the input rather than whatever value was being processed when it broke.
An app may also return a `NotCompleted` itself. If it supplies no source, the proxy's identity is filled in on the way out, so the failure is stored under the same unique ID a successful result would have had. If it supplies one, that is left alone — an app that knows a different file is at fault can say so.
cogent3-scinexus-b16bc06/docs/explanation/type-system.md 0000664 0000000 0000000 00000005534 15257614025 0023435 0 ustar 00root root 0000000 0000000 # Type system
!!! abstract ""
Why types are checked at composition time rather than call time, how `check_type_compatibility` works, handling of Union types and forward references, the role of `register_type_namespace`, and the relationship to `typeguard` for runtime checking.
## Composability rules
There are rules around app composition, starting with app types. Loaders and writers are special cases. If included, a loader must always be first, e.g.
```python { notest }
app = a_loader + a_generic
```
If included, a writer must always be last, e.g.
```python { notest }
app = a_generic + a_writer
```
Changing the order for either of the above will result in a `TypeError`.
The next constraint on app composition are the input and output types of the apps involved. Specifically, apps define the type of input they work on and the type of output they produce. For two apps to be composed, the output (or return) type of app on the left (e.g. `a_loader`) must overlap with the input type of the app on the right (e.g. `a_generic`). If they don't match, a `TypeError` is raised.
## Built-in type protocols and aliases
SciNexus defines two type-level constructs used across the framework:
- `SerialisableType` -- a `Protocol` that any object with a `to_rich_dict()` method satisfies. Database writer apps rely on this to serialise results before storing them in a data store.
- `IdentifierType` -- a type alias (`str | Path | DataMemberABC`) representing the accepted ways to identify a member of a data store. Loader apps accept this as input.
See the [API reference](../reference/utilities.md#type-system) for details.
## Runtime type checking
In addition to checking type compatibility when apps are composed, scinexus validates input data at runtime before each call to `main()`. This uses `typeguard.check_type` to verify that the data matches the app's declared input type. On a mismatch, a `NotCompleted` is returned with a message naming the received and expected types.
!!! important "Why this matters"
Without runtime type checking, passing the wrong data type to an app still fails — but the error occurs inside the users `main()` and can be confusing. For example, a message like `'NoneType' object has no attribute 'blah'` gives little indication that the real problem is a type mismatch from an upstream app. With runtime checking enabled, scinexus catches this before entering `main()` and reports the mismatch clearly.
### Disabling type checking with `check_data_type`
Runtime type checking is enabled by default. For mature pipelines where type correctness has been established, you can disable it to remove the small overhead of the `typeguard` check:
```python { notest }
app = read_json() + validate(required_field="name")
app.check_data_type = False
```
Setting `check_data_type` on the outermost app propagates the setting to all apps in the pipeline.
cogent3-scinexus-b16bc06/docs/explanation/why-composable-apps.md 0000664 0000000 0000000 00000007371 15257614025 0025025 0 ustar 00root root 0000000 0000000 # Why composable apps?
!!! abstract ""
The design philosophy behind `scinexus` and why composing single-purpose apps leads to more robust, reproducible scientific computation.
## Make your algorithms more robust
As the robustness of POSIX based operating systems (think Linux, Mac OS, Unix) can attest, writing algorithms that stitch together multiple single purpose applications is a *Very Good Thing*â„¢. This is most elegantly expressed as a part of the Unix design philosophy.
???+ quote
Write programs that do one thing and do it well. Write programs to work together.
— Doug McIlroy
**`scinexus` encourages this design pattern.** We leverage the Python type annotation system to govern the compatibility (composability) of different applications. This enables in-process composition of your applications with validation of the consistency of the pipeline and the consistency of the data being run through it.
We can expand on this slightly for the problem of scientific computation by considering the critical benchmark of satisfying the conditions for reproducible computation, i.e. the obligation to track all of the properties affecting the execution of your algorithm. Examples of this are the operating system, the language version, the seed used for the random number generator, etc.
**`scinexus` does this for you.** For example, we intercept all arguments (including default values) passed to the construction of apps and record them so that the app state is logged. If you, the developer, also leverage the capabilities of the [`scitrack`](https://pypi.org/project/scitrack/) logging package (which `scinexus` has as a dependency), you can capture extra information such as versions of packages that your application depends on. We provide an [example](../howto/log-and-cite.md#leveraging-scitrack-for-reproducibility) of using `scitrack` for these cases.
## Improve the accessibility of your work for end users
Apps are ready-made functions that users can run on their data without needing technical expertise. They’re easy for non-programmers to use, and can be linked together into pipelines. This lets users process one or millions of records at once—without writing loops, conditionals, or other structural code.
## When to use apps
Here are three examples of when to use apps.
### To make a callable class
You want a class to be callable, optionally with its input and output types checked at runtime, without writing the boilerplate yourself.
### To make a composite data transform
You want to chain several steps into a single transformation and reuse that combination across a code base. For example, converting an object to a JSON string and then compressing it involves two separate functions, plus possible configuration options for compression. By writing each step as an app, you can compose them into a single instance, assign it to a module-level variable. Importing and using that instance wherever it is needed ensures consistency in transformation behaviour.
!!! tip "Composite apps can themselves be used as attributes of other apps — apps can have apps as attributes 🤯🚀"
### To make a robust multi-step pipeline
You have a composite transformation that you want to apply to many data records. Written conventionally, this would require a loop that calls each step in turn, traps any exceptions, handles their consequences, and passes successful results on to the next step.
The `apply_to()` and `as_completed()` methods on composed apps handle this pattern for you. They iterate over the input records, run each one through the pipeline, and collect the results — so you do not need to write the loop, the exception handling, or the conditional checks between steps. And as an extra bonus, you can turn on parallelisation using a single argument.
cogent3-scinexus-b16bc06/docs/howto/ 0000775 0000000 0000000 00000000000 15257614025 0017417 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/docs/howto/customise-display-and-ids.md 0000664 0000000 0000000 00000014636 15257614025 0024746 0 ustar 00root root 0000000 0000000 # Customise display and IDs
!!! abstract ""
How to use `set_summary_display` to transform data store summary output into richer objects, and `set_id_from_source` to control how unique identifiers are extracted from data.
## Summary display default
By default, summary properties like `.describe` return Python primitive types like `dict` and `list`.
```python { linenums="1" notest }
from scinexus import open_data_store
dstore = open_data_store("data/raw.zip", suffix="fa", mode="r")
print(type(dstore.describe), "", dstore.describe, sep="\n")
{'completed': 1035, 'not_completed': 0, 'logs': 0}
```
## Customising summary display
You can register a customised display function for your project. For `cogent3`, it converts them into `cogent3` `Table` objects:
```python { linenums="1" notest }
from scinexus.data_store import set_summary_display
from cogent3.core.table import Table
def summary_to_table(data, *, name):
if isinstance(data, dict):
title = data.pop("title", name)
rows = [[k, v] for k, v in data.items()]
return Table(header=["Condition", "Value"], data=rows, title=title)
if isinstance(data, list):
if not data:
return Table(header=[], data=[], title=name)
header = list(data[0].keys())
rows = [list(row.values()) for row in data]
return Table(header=header, data=rows, title=name)
return data
set_summary_display(summary_to_table)
```
This results in the following:
```python { linenums="1" notest }
describe
======================
Condition Value
----------------------
completed 1035
not_completed 0
logs 0
----------------------
```
!!! note
`cogent3` registers this transformation automatically when you `import cogent3.app`, so you get `Table` output without any setup in cogent3 projects.
### Unsetting the display function
Reset the display function and revert to the default `scinexus` behaviour as follows:
```python { notest }
set_summary_display(None)
```
## Default unique ID extraction
Being able to extract unique identifiers for individual data objects is fundamental to the ability of scinexus to track provenance of individual results. Because of its roots from `cogent3`, the `scinexus` default `get_unique_id` function extracts this information from a `.source` attribute. That function, strips format suffixes from file names to derive unique keys for data store records.
```python { linenums="1" notest }
from scinexus import get_id_from_source
func = get_id_from_source()
print(func("gene_001.fasta.gz"), func("sample.txt"))
# gene_001 sample
```
## Customising unique ID extraction
Register a custom extractor when your naming convention differs:
```python { linenums="1" notest }
from scinexus.data_store import set_id_from_source, get_id_from_source
def extract_ensembl_id(data):
name = str(data)
if name.startswith("ENSG"):
return name.split(".")[0]
return name
set_id_from_source(extract_ensembl_id)
func = get_id_from_source()
# Now the registered function is used as the default
print(func("ENSG00000157184.fa"), func("gene_001.fasta.gz"))
# ENSG00000157184 gene_001.fasta.gz
```
## Reset to default
```python { linenums="1" notest }
from scinexus.data_store import set_id_from_source
set_id_from_source(None)
```
## Over-riding the default per-call
You can also override per-call without affecting the global default:
```python { notest }
result = app.apply_to(dstore, id_from_source=extract_ensembl_id)
```
cogent3-scinexus-b16bc06/docs/howto/extend-type-checking.md 0000664 0000000 0000000 00000010675 15257614025 0023771 0 ustar 00root root 0000000 0000000 # Extend type checking
!!! abstract ""
How to use `register_type_namespace` to make forward references from downstream packages resolvable at composition time, enabling third-party types in app pipelines.
## The problem
When you compose apps with `+`, `scinexus` checks that the output type of the left app is compatible with the input type of the right app. Type hints are often written as forward references — strings like `"Alignment"` or `"PhyloNode"` — to avoid circular imports. At composition time `scinexus` must resolve these strings to actual classes, but it only knows about its own types by default. If your package defines custom types used in app hints, `scinexus` cannot resolve them without help.
## The solution
`register_type_namespace` lets a downstream package register a **lazy namespace provider** — a zero-argument callable that returns a `dict[str, type]`. When `scinexus` encounters an unresolved forward reference, it queries each registered provider in order until it finds a match.
```python { notest }
from scinexus.typing import register_type_namespace
register_type_namespace(my_provider)
```
The provider is called lazily each time a name needs resolving, so the package can defer heavy imports. Providers are responsible for their own caching. Registration is idempotent: re-registering the same callable is a no-op.
???- example "How `cogent3` does it"
```python { linenums="1" notest }
from scinexus.typing import register_type_namespace
_resolution_ns = None
def _get_resolution_namespace():
global _resolution_ns
if _resolution_ns is not None:
return _resolution_ns
from cogent3.core.alignment import Alignment, SequenceCollection # (1)!
from cogent3.core.tree import PhyloNode
# ... other imports ...
_resolution_ns = {
"Alignment": Alignment,
"SequenceCollection": SequenceCollection,
"PhyloNode": PhyloNode, # (3)!
# ... other types ...
}
return _resolution_ns
register_type_namespace(_get_resolution_namespace) # (2)!
```
1. `cogent3` defines many types (`Alignment`, `PhyloNode`, `Table`, etc.) that are used as forward references in app type hints.
2. In `cogent3/app/typing.py`, a resolution namespace is built lazily and registered with `scinexus`
3. With this registration, any `scinexus` app that uses `"PhyloNode"` as a type hint will resolve correctly at composition time without the user importing `PhyloNode` explicitly.
## Registering your own package's types
Follow the same pattern: define a lazy provider function that imports and caches your types, then register it at module level.
```python { notest }
from scinexus.typing import register_type_namespace
_ns = None
def _get_my_types():
global _ns
if _ns is not None:
return _ns
from my_package.core import MyDataType, MyResultType
_ns = {
"MyDataType": MyDataType,
"MyResultType": MyResultType,
}
return _ns
register_type_namespace(_get_my_types)
```
Place this in a module that is imported early (e.g. your package's `typing.py` or `__init__.py`). Once registered, apps using `"MyDataType"` as a forward reference will resolve correctly when composed with other apps.
cogent3-scinexus-b16bc06/docs/howto/handle-failures.md 0000664 0000000 0000000 00000016505 15257614025 0023013 0 ustar 00root root 0000000 0000000 # Handle failures
!!! abstract ""
How to create `NotCompleted` values, check their truthiness, inspect their attributes, and control propagation with `skip_not_completed=False`.
## `NotCompleted` FALSE type
A FALSE type is returned when a condition is not met. For example, below we create an app that selects 2 specific sequences from an alignment. Applying this to a data set where a "Mouse" sequence does not exist produces a FALSE type.
```python { linenums="1" notest }
import cogent3 as c3
aln = c3.get_dataset("primate-brca1")
select_seqs = c3.get_app("take_named_seqs", "Mouse", "Human")
result = select_seqs(aln)
assert result == False
print(result)
# NotCompleted(type=FAIL, origin=take_named_seqs, source="brca1", message="named
# seq(s) {'Mouse'} not in ('FlyingLem', 'TreeShrew', 'Galago', 'HowlerMon',
# 'Rhesus', 'Orangutan', 'Gorilla', 'Chimpanzee', 'Human')")
```
## Inspecting `NotCompleted` attributes
The `NotCompleted` instance has attributes identifying what data failed:
```python { notest }
result.source
```
Where the failure occurred:
```python { notest }
result.origin
```
And the reason for the failure:
```python { notest }
result.message
```
The `.type` attribute is the `NotCompletedType` enum value (e.g. `NotCompletedType.FALSE`, `NotCompletedType.ERROR`, or `NotCompletedType.BUG`).
## `NotCompleted` ERROR type
An ERROR type is returned if an unexpected condition occurs, such as an exception raised during execution. Here we illustrate this by trying to open a file with an incorrect path.
???- example "Example"
```python { linenums="1" notest }
import cogent3 as c3
reader = c3.get_app("load_aligned", moltype="dna")
result = reader("primate_brca1.fasta")
print(result)
# NotCompleted(type=ERROR, origin=load_aligned, source="primate_brca1",
# message="Traceback (most recent call last): File
# "/Users/gavin/repos/SciNexus/src/scinexus/composable.py", line 545, in __call__
# result = self.main(val, *args, **kwargs) [...]
```
## Composed functions propagate `NotCompleted` results
If you have a composed function with multiple steps and a failure occurs, the resulting `NotCompleted` is returned without any of the subsequent steps being applied. For example, we make a composed app from both of the above apps:
???- example "Example"
```python { linenums="1" notest }
import cogent3 as c3
reader = c3.get_app("load_aligned", moltype="dna")
select_seqs = c3.get_app("take_named_seqs", "Mouse", "Human")
app = reader + select_seqs
result = app("data/primate_brca1.fasta")
print(result)
# NotCompleted(type=FAIL, origin=take_named_seqs, source="primate_brca1",
# message="named seq(s) {'Mouse'} not in ('FlyingLem', 'TreeShrew', 'Galago',
# 'HowlerMon', 'Rhesus', 'Orangutan', 'Gorilla', 'Chimpanzee', 'Human')")
```
The failure originated in `select_seqs` (an instance of `take_named_seqs`), and `reader` ran successfully — but the `NotCompleted` propagated through the rest of the pipeline.
???- example "Example"
```python { linenums="1" notest }
import cogent3 as c3
reader = c3.get_app("load_aligned", moltype="dna")
select_seqs = c3.get_app("take_named_seqs", "Mouse", "Human")
app = reader + select_seqs
result = app("primate_brca1.fasta")
print(result)
# NotCompleted(type=ERROR, origin=load_aligned, source="primate_brca1", [...]
```
Here the failure originated in `reader` (bad path), and `select_seqs` was never called.
## Creating `NotCompleted` in your own apps
You can return a `NotCompleted` from your own app to signal that a particular input cannot be processed:
```python { linenums="1" notest }
from scinexus import define_app, NotCompleted, NotCompletedType
@define_app
def require_min_length(val: str, min_length: int = 10) -> str:
if len(val) < min_length:
return NotCompleted(
NotCompletedType.FALSE,
"require_min_length",
val,
message=f"too short: {len(val)} < {min_length}",
)
return val
```
## Receiving `NotCompleted` with `skip_not_completed=False`
By default, apps skip `NotCompleted` inputs — they propagate without calling `main()`. If your app needs to see `NotCompleted` values, set `skip_not_completed=False`:
```python { linenums="1" notest }
from scinexus import define_app, NotCompleted
@define_app(skip_not_completed=False)
def log_failures(val: str) -> str:
if isinstance(val, NotCompleted):
print(f"Failure: {val.message}")
return val
return val
```
!!! note "Writer apps get this for free"
A writer never skips `NotCompleted`, whatever you pass for `skip_not_completed`. `apply_to()` invokes a writer's `main()` directly rather than through the machinery that does the skipping, so the flag could not be honoured there. It is forced off for writers so that a one-off `app(data)` call behaves the same way.
The practical consequence is that a writer's `main()` must handle `NotCompleted` itself, typically by sending it to `write_not_completed()`. Its first parameter annotation does not constrain what arrives.
cogent3-scinexus-b16bc06/docs/howto/index.md 0000664 0000000 0000000 00000002226 15257614025 0021052 0 ustar 00root root 0000000 0000000 # How-to guides
Task-oriented recipes for common `scinexus` operations, each self-contained.
- [Write a function app](write-a-function-app.md) -- use `@define_app` on a function
- [Write a class app](write-a-class-app.md) -- use `@define_app` on a class with `main()`
- [Handle failures](handle-failures.md) -- create and inspect `NotCompleted` values
- [Use data stores](use-data-stores.md) -- open, read, and write data stores
- [Read and write files](read-and-write-files.md) -- IO with compression, atomic writes, streaming
- [Run in parallel](run-in-parallel.md) -- parallel execution with pluggable backends
- [Track progress](track-progress.md) -- progress bars with `tqdm` or `rich`
- [Log and cite](log-and-cite.md) -- `scitrack` logging and citation tracking
- [Customise display and IDs](customise-display-and-ids.md) -- `set_summary_display` and `set_id_from_source`
- [Extend type checking](extend-type-checking.md) -- register type namespaces for downstream packages
- [Use with a coding agent](use-with-coding-agents.md) -- install the `scinexus` agent skill
- [Migrate from cogent3](migrate-from-cogent3.md) -- update code from `cogent3.app` to `scinexus`
cogent3-scinexus-b16bc06/docs/howto/log-and-cite.md 0000664 0000000 0000000 00000017744 15257614025 0022221 0 ustar 00root root 0000000 0000000 # Log and cite
!!! abstract ""
How to use `scitrack` logging in apps, control logging in `apply_to`, and access citation records from composed pipelines.
## Leveraging `scitrack` for reproducibility
We reproduce here one of the examples from [scitrack](https://github.com/HuttleyLab/scitrack).
???- example "Using `scitrack` in a `click` app"
```python linenums="1"
import click
from scitrack import CachingLogger
LOGGER = CachingLogger()
@click.command()
@click.option("-i", "--infile", type=click.Path(exists=True))
@click.option("-t", "--test", is_flag=True, help="Run test.")
def main(infile, test):
# capture the local variables, at this point just provided arguments
LOGGER.log_args() # (1)!
LOGGER.log_versions("numpy") # (2)!
LOGGER.input_file(infile) # (3)!
LOGGER.log_file_path = "some_path.log" # (4)!
if __name__ == "__main__":
main()
```
1. :man_raising_hand: A single statement and you have captured all the input arguments and their values, including defaults!
2. This captures the version numbers of the packages our application depends on.
3. This logs the path to `infile` and its md5sum.
4. Until you assign the path where you want the file written, this content has been cached.
## Controlling logging in `apply_to`
By default, `apply_to` creates a `CachingLogger` that records the composable function, package versions, output paths, MD5 checksums of every result, and total elapsed time. The log is then written into the output data store. This is the recommended setting for production analyses because it gives you a complete, self-contained record of what ran and what it produced.
```python { notest }
result = process.apply_to(dstore) # logger=True by default
```
You can also pass your own `CachingLogger` instance if you want to configure it beforehand or reuse one across multiple calls.
```python { notest }
from scitrack import CachingLogger
LOGGER = CachingLogger()
LOGGER.log_args()
result = process.apply_to(dstore, logger=LOGGER)
```
### Disabling logging
Set `logger=False` to skip logging entirely.
```python { notest }
result = process.apply_to(dstore, logger=False)
```
This is useful when:
- **Your project is small** and a full provenance log is unnecessary.
- **Logging is handled externally**, for example by a workflow manager or your own `CachingLogger` that wraps several `apply_to` calls.
- **You want to avoid the overhead** of computing an MD5 checksum for every result object, which can be noticeable for large or numerous outputs.
## Make it easy for your work to be cited
Correctly attributing the authors of algorithms and software is a requirement of good scientific practice. `scinexus` makes this easy by letting app authors declare citations that are automatically tracked through composed pipelines.
Use the `cite` parameter of `define_app` (or the base classes) to attach a citation. The `citeable` library provides several classes for this purpose.
???- example "Adding a citation to your app"
```python { linenums="1" notest }
from citeable import Software
from scinexus import define_app
from cogent3.app.typing import AlignedSeqsType
from cogent3 import get_app
my_cite = Software(
author=["Doe, J", "Smith, A"],
title="My Sequence Filter",
year=2025,
url="https://example.com/my-filter",
version="0.1.0",
)
@define_app(cite=my_cite) # (1)!
def strict_filter(val: AlignedSeqsType) -> AlignedSeqsType:
"""Remove sequences shorter than the alignment."""
return val.omit_bad_seqs()
app = strict_filter()
loader = get_app("load_aligned", moltype="dna", format_name="fasta")
pipeline = loader + strict_filter()
print(pipeline.citations) # (2)!
print(f"\n{pipeline.bib}") # (3)!
# (Software( author=['Doe, J', 'Smith, A'], title='My Sequence Filter',
# year=2025, version='0.1.0', url='https://example.com/my-filter', [...]
```
1. Use the `cite` parameter of `define_app` to attach a citation
2. The `.citations` property returns citations as a tuple. When apps are composed into a pipeline, `.citations` collects unique citations from all apps in the chain.
3. The `.bib` gives the BibTeX string.
## Extracting citations from a data store
When a composed pipeline is run via `apply_to()`, citations are automatically saved in the output data store.
???- example "Citations in data stores"
```python { linenums="1" notest }
from citeable import Software
from scinexus import define_app, open_data_store
from cogent3.app.typing import AlignedSeqsType
from cogent3 import get_app
my_cite = Software(
author=["Doe, J"],
title="My Sequence Filter",
year=2025,
)
@define_app(cite=my_cite)
def strict_filter(val: AlignedSeqsType) -> AlignedSeqsType:
return val.omit_bad_seqs()
in_dstore = open_data_store("data/raw.zip", suffix="fa", limit=5)
out_dstore = open_data_store("cited_results", suffix="fa", mode="w")
loader = get_app("load_aligned", moltype="dna", format_name="fasta")
writer = get_app("write_seqs", data_store=out_dstore, format_name="fasta")
process = loader + strict_filter() + writer
result = process.apply_to(in_dstore)
result.summary_citations # (1)!
result.write_bib("my_analysis.bib") # (2)!
```
1. Because we are using `cogent3`, the property returns a `cogent3` `Table` of all citations stored in the data store.
2. You can export to a BibTeX file.
!!! note
`ReadOnlyDataStoreZipped` supports reading stored citations but not writing them.
cogent3-scinexus-b16bc06/docs/howto/migrate-from-cogent3.md 0000664 0000000 0000000 00000005636 15257614025 0023704 0 ustar 00root root 0000000 0000000 # Migrate from cogent3
!!! abstract ""
A guide for users moving from `cogent3.app` to `scinexus`.
## Update imports
Replace `cogent3.app` imports with their `scinexus` equivalents:
```python { notest }
# before
from cogent3.app.composable import define_app, NotCompleted
# after
from scinexus import define_app, NotCompleted
```
## Summary methods return plain Python objects
Data store summary methods (`describe`, `summary_logs`, `summary_not_completed`, `summary_citations`, `validate`) now return `list[dict]` or `dict` instead of cogent3 `Table` objects.
To restore `Table`-based display, register a converter with `set_summary_display()`:
```python { notest }
from cogent3.core.table import Table
from scinexus.data_store import set_summary_display
def _summary_to_table(data, *, name):
if isinstance(data, dict):
rows = [[k, v] for k, v in data.items()]
return Table(header=["Condition", "Value"], data=rows, title=name)
if isinstance(data, list) and data:
header = list(data[0].keys())
rows = [list(row.values()) for row in data]
return Table(header=header, data=rows, title=name)
return data
set_summary_display(_summary_to_table)
```
## NotCompletedType is now an enum
Failure types are categorised using the `NotCompletedType` enum rather than bare strings:
```python { notest }
from scinexus.composable import NotCompletedType
# before
nc.type == "ERROR"
# after
nc.type == NotCompletedType.ERROR
```
The three values are `ERROR`, `FAIL`, and `BUG`.
## App composition uses shallow copies
Composing apps with `+` now creates a shallow copy of the right-hand operand. Composed pipelines no longer share mutable state, so `ComposableApp.disconnect()` is no longer needed and is deprecated.
## New features
These capabilities are new in `scinexus` and were not available in `cogent3.app`:
- **`check_data_type` property** -- toggle runtime input type checking on or off. See [Runtime type checking](../explanation/type-system.md#runtime-type-checking).
- **Better IDE integration through static typing support** -- `AppBase[T, R]`, `ComposableApp[T, R]`, and `WriterApp[T, R]` can be inherited from directly as an alternative to `@define_app`.
- **Pluggable parallel backends** -- choose between stdlib multiprocessing, stdlib threads, loky, or MPI backends. See [Run in parallel](run-in-parallel.md).
- **Pluggable progress bars** -- use `tqdm` or `rich` for progress display. See [Track progress](track-progress.md).
- **Custom identifier extraction** -- register a custom function for extracting storage identifiers from data via `set_id_from_source()`. See [Customise display and IDs](customise-display-and-ids.md).
- **Logging can be disabled** -- pass `logger=False` to `apply_to()` to skip log file creation.
## Full changelog
See the [changelog](https://github.com/cogent3/scinexus/blob/main/changelog.md) for a complete list of changes from the cogent3 app infrastructure.
cogent3-scinexus-b16bc06/docs/howto/read-and-write-files.md 0000664 0000000 0000000 00000013550 15257614025 0023650 0 ustar 00root root 0000000 0000000 # Read and write files
!!! abstract ""
How to use `open_()` for reading and writing files with automatic compression detection (gzip, bzip2, lzma, zip), `atomic_write` for safe file writes that clean up on failure, `iter_splitlines` and `iter_line_blocks` for streaming large files, and `is_url`/`open_url` for working with URLs.
## Writing a compressed file
`open_()` detects the compression format from the file suffix and handles it automatically. Writing a gzip-compressed text file is identical to writing a plain text file — just use a `.gz` suffix.
```python { linenums="1" notest }
from scinexus import open_
with open_("data/sample.txt.gz", "wt") as f:
f.write("Hello, compressed world!\n")
f.write("Line two of the file.\n")
```
## Reading a compressed file
Reading works the same way — `open_()` detects the `.gz` suffix and decompresses transparently.
```python { linenums="1" notest }
from scinexus import open_
with open_("data/sample.txt.gz") as f:
print(f.read())
# Hello, compressed world! Line two of the file.
```
Supported compression formats are gzip (`.gz`), bzip2 (`.bz2`), lzma (`.xz`, `.lzma`), and zip (`.zip`).
## Reading a URL
`open_()` also handles URLs. Use `is_url()` to check whether a path is a URL before opening it.
???- example "Checking and reading a URL"
```python linenums="1"
from scinexus.io_util import is_url
from scinexus import open_
url = "https://github.com/user-attachments/files/26728407/raw.zip"
print(is_url(url)) # (1)!
with open_(url, "rb") as f: # (2)!
header = f.read(20)
print(header)
```
1. `is_url()` returns `True` for `http`, `https`, and `file` scheme URLs.
2. `open_()` detects the URL and delegates to `open_url()`. Only read mode is supported for URLs.
## Efficiently reading large files
Reading an entire large file into memory or iterating line by line with Python's built-in `readline()` can be inefficient. The built-in approach makes a system call for every line, which becomes a bottleneck for files with millions of lines. `scinexus` provides two functions that read data in large chunks and then split into lines, greatly reducing I/O overhead.
### `iter_splitlines`
`iter_splitlines(path, chunk_size=1_000_000, *, as_bytes=False)` reads a file in chunks (default 1 MB) and yields individual lines. It correctly handles lines that span chunk boundaries. An empty last line is never yielded, so a file ending on a newline gives the same lines as one that does not.
```python { notest }
from scinexus.io_util import iter_splitlines
for line in iter_splitlines("large_file.txt"):
process(line)
```
Pass `as_bytes=True` to open the file in binary mode and get `bytes` lines, skipping the decoding step. The argument is keyword-only.
```python { notest }
from scinexus.io_util import iter_splitlines
for line in iter_splitlines("large_file.txt", as_bytes=True):
process(line) # line is bytes, e.g. b"first line"
```
The two modes do not always split a file into the same number of lines. In text mode the file is read with universal newlines, so `\r`, `\n` and `\r\n` all become line breaks, and `str.splitlines()` breaks on a further eight characters including vertical tab and form feed. In binary mode there is no translation and `bytes.splitlines()` breaks only on `\r`, `\n` and `\r\n`:
```python { notest }
from pathlib import Path
from scinexus.io_util import iter_splitlines
Path("pages.txt").write_bytes(b"alpha\x0cbeta\n")
list(iter_splitlines("pages.txt")) # ['alpha', 'beta']
list(iter_splitlines("pages.txt", as_bytes=True)) # [b'alpha\x0cbeta']
```
For compressed files `as_bytes=True` gives the decompressed bytes, not the raw compressed ones.
### `iter_line_blocks`
`iter_line_blocks(path, num_lines=1000, chunk_size=5_000_000, *, as_bytes=False)` builds on `iter_splitlines` — it accumulates lines into lists of `num_lines` and yields each list. This is useful when downstream processing works on batches (e.g. FASTA records where each record spans a fixed number of lines).
```python { notest }
from scinexus.io_util import iter_line_blocks
for block in iter_line_blocks("large_file.txt", num_lines=1000):
process_batch(block) # block is a list of up to 1000 strings
```
`iter_line_blocks` passes `as_bytes` straight through, so each block is a list of `bytes` and everything said above about the two modes applies here too. `num_lines` is unaffected, blocks are still at most `num_lines` long.
```python { notest }
for block in iter_line_blocks("large_file.txt", num_lines=1000, as_bytes=True):
process_batch(block) # block is a list of up to 1000 bytes objects
```
Use `iter_splitlines` when you need one line at a time. Use `iter_line_blocks` when your processing naturally operates on batches of lines.
cogent3-scinexus-b16bc06/docs/howto/run-in-parallel.md 0000664 0000000 0000000 00000024255 15257614025 0022753 0 ustar 00root root 0000000 0000000 # Run in parallel
!!! abstract ""
How to choose your preferred parallelisation backend, how to parallelise any function using `parallel.map`, `parallel.imap`, and `parallel.as_completed` as standalone utilities, and how to enable parallel execution in app pipelines with `parallel=True` and `par_kw`.
## Data level parallelism
`scinexus` supports parallel computation for the common case where the same calculation needs to be applied to many independent data items. The master splits the work among available CPU cores, each worker processes its share, and results are collected. Whether a worker is a process or a thread depends on the backend, and on a free-threaded build the default is threads.
!!! warning
Parallelism is not always faster. You should see a performance gain when the computation time per task significantly exceeds the overhead of distributing work. If individual tasks are very fast, the overhead of distributing them can dominate — for the process backends that overhead includes pickling every argument and result and sending it between processes.
If individual output files are small, storing results in a single file (e.g. a `.sqlitedb` database) is more efficient than writing many small files.
## Choosing a parallel backend
`scinexus` supports four parallel backends. Two of them use only the Python standard library and require no extra installs.
| Backend | Install | Best for |
|---|---|---|
| `"multiprocess"` | included | scripts, CI, environments where you control dependencies |
| `"threads"` | included | free-threaded (no-GIL) builds, where it is the default |
| `"loky"` | `pip install "scinexus[loky]"` | Jupyter notebooks, interactive sessions, long-running pools |
| `"mpi"` | `pip install "scinexus[mpi]"` | HPC clusters with multiple nodes |
Set the backend once, typically at the top of your script or notebook:
```python { notest }
import scinexus
scinexus.set_parallel_backend("loky")
```
!!! note
The `"loky"` backend uses [loky](https://loky.readthedocs.io/) which provides reusable process pools and robust pickling via `cloudpickle`. This makes it the recommended choice for Jupyter notebooks, where the stdlib `ProcessPoolExecutor` can fail to serialise closures and lambda functions.
### If you do not choose, one is chosen for you
With no call to `set_parallel_backend`, the backend is `"threads"` when the GIL is not in force **and** the caller is not itself a pool worker, and `"multiprocess"` in every other case. On a standard CPython build you therefore get processes, and on a free-threaded (no-GIL) build you get threads.
Two things besides the build can put the GIL back in force: setting `PYTHON_GIL=1`, and importing an extension module that does not declare free-threading support. The second can happen partway through a run, so the choice is revisited on every call to `imap`, `map` or `as_completed`. Specifying the backend using `set_parallel_backend` is how you opt out of threads on a free-threaded build.
```python { notest }
import scinexus
scinexus.set_parallel_backend("multiprocess") # processes, whatever the build
```
!!! note "What pinning does not cover"
The setting applies to primary process alone, so a worker does not inherit it and chooses again for itself. Passing `use_mpi=True` to `imap`, `map` or `as_completed` uses MPI for that call whatever you pinned. And `get_rank()`, `get_size()` and `is_master_process()` report on the context actually running the current thread rather than on the default you set.
### What crosses to a worker
The process backends — `"multiprocess"`, `"loky"` and `"mpi"` — pickle the function and its arguments and send them to another process. Each worker gets its own copy of your app, so mutating `self` inside `main()` does not propagate, and closures and lambdas are refused[^1].
[^1]: `"loky"` is the most forgiving, since it pickles via `cloudpickle`
The `"threads"` backend sends nothing. Workers share the app instance and everything reachable from it, so closures and lambdas are accepted, but a `main()` that mutates `self` or writes module-level state — the `numpy.random` global generator, say — is a data race here where it was harmless under the process backends.
### Getting a specific backend without changing the default
If your code requires a particular backend, pass the ``backend`` argument to ``get_parallel_backend``. This returns an instance of the requested backend without changing the global default, so other packages that depend on the current setting are unaffected:
```python { notest }
from scinexus import get_parallel_backend
backend = get_parallel_backend(backend="loky")
```
## Parallel computation on a single computer
### Using `app.apply_to()`
If you have a composed app **with** a writer, use `apply_to()` with the `parallel` and `par_kw` keyword arguments:
```python { notest }
result = app.apply_to(dstore, parallel=True, par_kw=dict(max_workers=4))
```
### Using `app.as_completed()`
If you have a composed app **without** a writer, use `as_completed()`. This returns a generator, so wrap it with `list()` or iterate over it:
```python { notest }
results = list(app.as_completed(dstore, parallel=True, par_kw=dict(max_workers=4)))
```
### Using `scinexus.parallel` directly
For parallelising any function (not just apps), use the functions in `scinexus.parallel`.
#### `parallel.as_completed` -- results in completion order
Returns results as they finish. The order may differ from the input order. It also tends to balance work better across compute nodes than `imap` or `map`.
```python { notest }
from scinexus import parallel
result = list(parallel.as_completed(is_prime, PRIMES, max_workers=4))
```
The first argument is the function to call, the second is the iterable of inputs. Each input element is passed as a single argument to the function, and `as_completed` submits one task per item on every backend. If you want the work batched, use `imap` on a process or MPI backend, which chunks by `chunksize`.
!!! note
If you don't specify `max_workers`, all available CPUs are used. Under MPI it instead means the workers the job was launched with, which is not something `max_workers` can change. `"threads"` `chunksize` but ignores it.
#### `parallel.imap` -- preserving input order (generator)
Returns results in the same order as the input, yielding one at a time:
```python { notest }
from scinexus import parallel
for result in parallel.imap(process_item, items, max_workers=4):
handle(result)
```
#### `parallel.map` -- preserving input order (list)
Same as `imap` but returns a list:
```python { notest }
from scinexus import parallel
results = parallel.map(process_item, items, max_workers=4)
```
### Complete example
```python { notest }
import math
from scinexus import parallel
def is_prime(n):
if n % 2 == 0:
return False
sqrt_n = int(math.floor(math.sqrt(n)))
for i in range(3, sqrt_n + 1, 2):
if n % i == 0:
return False
return True
PRIMES = [
112272535095293,
112582705942171,
115280095190773,
115797848077099,
117450548693743,
993960000099397,
]
if __name__ == "__main__":
results = parallel.map(is_prime, PRIMES, max_workers=4)
for number, prime in zip(PRIMES, results):
print(f"{number} is prime: {prime}")
```
## Parallel computation on multiple computers (MPI)
On systems with multiple nodes (e.g. an HPC cluster), use MPI via the [mpi4py](https://mpi4py.readthedocs.io/) library. You need to install an MPI implementation (e.g. [OpenMPI](https://www.open-mpi.org/)) and the `mpi4py` Python package
```bash
pip install mpi4py
```
or installing `scinexus` [with `mpi` extra](../install.md#optional-extras).
Set the backend to MPI:
```python { notest }
import scinexus
scinexus.set_parallel_backend("mpi")
```
Or pass `use_mpi=True` to any of the parallel functions:
```python { notest }
from scinexus import parallel
results = parallel.map(is_prime, PRIMES, use_mpi=True)
```
Or with app pipelines:
```python { notest }
result = app.apply_to(dstore, parallel=True, par_kw=dict(use_mpi=True))
```
To run an MPI script, invoke it via `mpiexec`:
```bash
mpiexec -n $PBS_NCPUS python3 -m mpi4py.futures my_script.py
```
!!! warning "Do not set `max_workers` under MPI"
Note that `max_workers` is absent from the calls above. The worker count is fixed by `mpiexec -n` before your program starts, so `max_workers` cannot change it and passing one that disagrees only earns a warning. Rank 0 is the master and does no work, so `-n $PBS_NCPUS` gives you `$PBS_NCPUS - 1` workers. See [How the MPI backend differs](../explanation/mpi-backend.md).
!!! note
You can use MPI for parallel execution on a single computer too. This can be useful for testing your code locally before migrating to a larger system.
### MPI script structure
MPI scripts must guard the main logic behind `if __name__ == "__main__":`:
```python { notest }
from scinexus import parallel
def process(data): ...
if __name__ == "__main__":
results = parallel.map(process, my_data, use_mpi=True)
```
## Custom backends
You can integrate any parallel engine by subclassing `Parallel`:
```python { notest }
from scinexus.parallel import Parallel, set_parallel_backend
class DaskBackend(Parallel):
def __init__(self, client):
self._client = client
def imap(self, f, s, max_workers=None, **kwargs):
futures = self._client.map(f, list(s))
yield from self._client.gather(futures)
def as_completed(self, f, s, max_workers=None, **kwargs):
from dask.distributed import as_completed
futures = self._client.map(f, list(s))
for future in as_completed(futures):
yield future.result()
def is_master_process(self):
from dask.distributed import get_worker
try:
get_worker()
return False
except ValueError:
return True
def get_rank(self):
return 0
def get_size(self):
return sum(self._client.nthreads().values())
set_parallel_backend(DaskBackend(client))
```
!!! warning "The above is a hypothetical wrapping of dask!"
The example for adding [dask](https://www.dask.org) as a parallelisation backend is not tested! Also note that this would require installing `"dask[distributed]"`.
cogent3-scinexus-b16bc06/docs/howto/track-progress.md 0000664 0000000 0000000 00000014462 15257614025 0022716 0 ustar 00root root 0000000 0000000 # Track progress
!!! abstract ""
How to choose your preferred progress bar backend and customise progress bars.
`scinexus` defaults to using the [tqdm](https://pypi.org/project/tqdm/) for progress bars. These behave well across terminal and notebook environments. We also support using [rich](https://pypi.org/project/rich/) for its progress bars. A single API for different progress backends.
## Choosing the progress bar backend
Use `set_progress_backend` to switch between backends. The default is `tqdm`.
```python { notest }
import scinexus
scinexus.set_progress_backend("rich") # switch to rich
scinexus.set_progress_backend("tqdm") # switch back to tqdm
scinexus.set_progress_backend(None) # reset to default (tqdm)
```
## Getting a progress bar
Use `get_progress` to obtain a `Progress` instance. Passing `show_progress=True` returns the current default backend.
```python { notest }
import scinexus
pbar = scinexus.get_progress(show_progress=True)
for item in pbar(range(100), msg="Processing"):
pass # your work here
```
You can pass keyword arguments to configure the default backend:
```python { notest }
import scinexus
pbar = scinexus.get_progress(show_progress=True, colour="blue", leave=True)
```
You can also pass a `Progress` instance directly:
```python { notest }
from scinexus.progress import RichProgress
pbar = scinexus.get_progress(show_progress=RichProgress())
```
!!! note
If you call `get_progress(show_progress=False)`, it returns `NoProgress`, which silently passes through the iterable.
## Nesting progress bars
=== "Using `tqdm` (default)"
Create nested progress bars using `child()`. Each bar can have its own description via the `msg` keyword. Create the child once before the loop — it automatically resets to zero on each subsequent call.
```python { notest }
import scinexus
pbar = scinexus.get_progress(show_progress=True)
child = pbar.child()
for batch in pbar(range(3), msg="Outer loop"):
for item in child(range(10), msg=f"Inner batch {batch}"):
pass # your work here
```
=== "Using `rich`"
The same nesting pattern works with the `rich` backend:
```python { notest }
import scinexus
scinexus.set_progress_backend("rich")
pbar = scinexus.get_progress(show_progress=True)
child = pbar.child()
for batch in pbar(range(3), msg="Outer loop"):
for item in child(range(10), msg=f"Inner batch {batch}"):
pass # your work here
```
`rich` children share the same `rich.progress.Progress` display instance, so all bars render together in a single live display.
The outer bar tracks the top-level iteration. Each call to `child()` creates a new `Progress` at the next cursor position, so inner bars appear below the outer one. The child bar is reused across iterations — on the second and subsequent calls, the bar resets to zero instead of creating a new one.
#### Push-based sub-contexts
When you need to report fractional progress rather than iterating, use `context()`:
```python { notest }
import scinexus
pbar = scinexus.get_progress(show_progress=True)
child = pbar.child()
for batch in pbar(range(3), msg="Processing"):
with child.context(msg=f"Batch {batch}") as ctx:
for i in range(100):
ctx.update(progress=i / 100, msg=f"Step {i}")
```
The context maps progress values from `[0.0, 1.0]` to the configured `[start, end]` range and is cleaned up automatically when the `with` block exits.
## Cleaning up
Both `Progress` and `ProgressContext` support the context manager protocol. Using a progress bar as a context manager ensures that `close()` is called automatically, which finalises the display and moves the cursor past the bars. Without cleanup, leftover bars can leave the terminal cursor in the wrong position.
=== "Using `tqdm` (default)"
```python
import scinexus
with scinexus.get_progress(show_progress=True) as pbar: # (1)!
child = pbar.child()
for batch in pbar(range(3), msg="Outer"):
for item in child(range(10), msg=f"Batch {batch}"):
pass
```
1. `close()` is called automatically and the cursor position is restored.
=== "Using `rich`"
```python
import scinexus
scinexus.set_progress_backend("rich")
with scinexus.get_progress(show_progress=True) as pbar: # (1)!
child = pbar.child()
for batch in pbar(range(3), msg="Outer"):
for item in child(range(10), msg=f"Batch {batch}"):
pass
```
1. `close()` is called automatically and the cursor position is restored.
???- tip "No context manager? No problem!"
```python
import scinexus
pbar = scinexus.get_progress(show_progress=True)
child = pbar.child()
for batch in pbar(range(3), msg="Outer"):
for item in child(range(10), msg=f"Batch {batch}"):
pass
pbar.close() # (1)!
```
1. Call `close()` explicitly when you are done
!!! note
Calling `close()` on a `Progress` instance also closes all of its children. For standalone `ProgressContext` objects (from `context()`), use the `with` statement as shown in the [push-based sub-contexts](#push-based-sub-contexts) section.
## Customising appearance
### Persisting bars after completion
By default, `tqdm` keeps the outermost bar visible after completion but clears nested bars. `rich` removes all bars. Use `leave` to control this:
```python { notest }
from scinexus.progress import TqdmProgress, RichProgress
# Keep all tqdm bars visible after completion
pbar = TqdmProgress(leave=True)
# Keep all rich bars visible after completion
pbar = RichProgress(leave=True)
```
You can also set `leave` independently on child bars:
```python { notest }
from scinexus.progress import TqdmProgress
pbar = TqdmProgress(leave=True)
child = pbar.child(leave=False) # child bars disappear, outer persists
for batch in pbar(range(3), msg="Outer"):
for item in child(range(10), msg=f"Batch {batch}"):
pass
```
### Setting bar colour
Both backends support a `colour` parameter. For `tqdm`, this sets the bar colour directly. For `rich`, it styles the bar column when the display is auto-created.
```python { notest }
from scinexus.progress import TqdmProgress, RichProgress
pbar = TqdmProgress(colour="green")
pbar = RichProgress(colour="cyan")
```
Colour is inherited by child bars.
cogent3-scinexus-b16bc06/docs/howto/use-data-stores.md 0000664 0000000 0000000 00000030104 15257614025 0022757 0 ustar 00root root 0000000 0000000 # Use data stores
!!! abstract ""
How to use `open_data_store` in read, write, and append modes with directory, zip, and SQLite backends, iterate over members, and inspect `.completed`, `.not_completed`, and `.summary_`.
## How do I use a data store?
A data store is just a "container". To open a data store you use the `open_data_store()` function. To load the data for a member of a data store you need an appropriately selected loader type of app.
## Supported operations on a data store
All data store classes can be iterated over, indexed, checked for membership. These operations return a `DataMember` object. In addition to providing access to members, the data store classes have convenience methods for describing their contents and providing summaries of log files that are included and of the `NotCompleted` members (see not completed).
## Opening a data store
Use the `open_data_store()` function, illustrated below. Use the mode argument to identify whether to open as read only (`mode="r"`), write (`mode="w"`) or append(`mode="a"`).
### Opening a read only data store
We open the zipped directory described above, defining the filenames ending in ``.fa`` as the data store members. All files within the directory become members of the data store (unless we use the ``limit`` argument).
```python { linenums="1" notest }
from scinexus import open_data_store
dstore = open_data_store("data/raw.zip", suffix="fa", mode="r") # (1)!
print(dstore)
dstore.describe # (2)!
m = dstore[0] # (3)!
for m in dstore[:5]: # (4)!
print(m)
m.read()[:20] # (5)!
# 1035x member
# ReadOnlyDataStoreZipped(source='/Users/gavin/repos/SciNexus/docs/data/raw.zip',
# members=[DataMember(data_store=/Users/gavin/repos/SciNexus/docs/data/raw.zip,
# unique_id=ENSG00000157184.fa),
# DataMember(data_store=/Users/gavin/repos/SciNexus/docs/data/raw.zip,
# unique_id=ENSG00000131791.fa)]...) ENSG00000157184.fa ENSG00000131791.fa
# ENSG00000127054.fa ENSG00000067704.fa ENSG00000182004.fa
```
1. Open a data store.
2. The `.describe` property summarises the contents.
3. You can index like any Python sequence.
4. Or loop over members.
5. And read data from a member.
!!! note
For a `DataStoreSqlite` member, the default data storage format is bytes. So reading the content of an individual record is best done using the `load_db` app.
### Making a writeable data store
The creation of a writeable data store is specified with `mode="w"`, or (to append) `mode="a"`. In the former case, any existing records are overwritten. In the latter case, existing records are ignored.
In a directory store, the `suffix` you open with names every *completed* record it writes. The identifier you pass to `write()` supplies the stem only, so `"brca1"`, `"brca1.fa"` and `"brca1.genbank"` all become `brca1.fasta` in a store opened with `suffix="fasta"`. Not-completed records, logs and checksums are kept in their own subdirectories under their own extensions, and the store's suffix does not apply to them.
!!! warning "A compression suffix is refused, not replaced"
Compression is the one part of the name that says how to read the record back, so a directory store will not quietly swap it. If the identifier names a compression the store does not write, `write()` raises `ValueError` rather than storing the record under a different name.
This part is specific to directory stores, since a SQLite store has no suffix to contradict. The rule that an identifier must name a record — a non-blank, non-hidden stem — applies to both.
```python { notest }
dstore = open_data_store("results", suffix="fasta", mode="w")
dstore.write(unique_id="brca1.fasta.gz", data=seqs)
# ValueError: identifier 'brca1.fasta.gz' names .gz, but a record
# stored as .fasta carries no compression
```
Open the store with the compression in its suffix instead, and its completed records are gzipped:
```python { notest }
dstore = open_data_store("results", suffix="fasta.gz", mode="w")
dstore.write(unique_id="brca1", data=seqs) # brca1.fasta.gz, gzipped
```
Not-completed records are always plain `.json`, whatever the store's own suffix is, so a compressed identifier is refused there too — including in a `suffix="fasta.gz"` store, where `write()` accepts `"brca1.fasta.gz"` and `write_not_completed()` does not.
## `DataStoreSqlite` stores serialised data
When you specify a Sqlitedb data store as your output (by using `open_data_store()`) you write multiple records into a single file making distribution easier.
!!! warning
The process which creates a Sqlitedb "locks" the file. If that process exits unnaturally (e.g. the run that was producing it was interrupted) then the file may remain in a locked state. If the db is in this state, `scinexus` will not modify it unless you explicitly unlock it.
### Closing a Sqlitedb data store
The lock is released by `close()`, and only by `close()`. That is what gives a lock you find on a file its meaning: it says the session that took it did not finish. So call `close()` once you have finished with a writable store, after reading whatever you need from it.
```python { notest }
out_dstore = open_data_store("results.sqlitedb", mode="w")
# ... write to it, then read your summaries from it ...
out_dstore.close()
```
A store that is garbage collected without being closed warns you and names the file, because it has left a lock behind that the next run will refuse to write over. Closing a data store ends access to it – trying to read from one afterwards raises an exception rather than returning stale answers.
Directory and zip data stores take no such lock, and closing one does nothing. `close()` is defined on every data store so that code holding whatever `open_data_store()` returned can close it without first asking which kind it got.
This is represented in the display as shown below.
```python { linenums="1" notest }
{"completed": 175, "not_completed": 0, "logs": 1, "title": "Unlocked db store."}
```
To unlock, open the store in a writable mode and execute the following:
```python { notest }
dstore = open_data_store("data/demo-locked.sqlitedb", mode="a")
dstore.unlock(force=True)
```
The store has to be writable, since unlocking writes to it (a read only store ignores the call). `force=True` is needed when the lock was taken by another process, which is the usual case for a store left behind by an interrupted run. Overriding it is meant to be a deliberate act, because the records in such a store were never confirmed complete.
## Interrogating run logs
If you use the `apply_to()` method, a `scitrack` logfile will be stored in the data store. This includes useful information regarding the run conditions that produced the contents of the data store.
```python { linenums="1" notest }
# [{'time': '2019-07-24 14:42:56', 'name': 'logs/load_unaligned-progressive_align-
# write_db-pid8650.log', 'python_version': '3.7.3', 'who': 'gavin', 'command':
# '/Users/gavin/miniconda3/envs/c3dev/lib/python3.7/site-
# packages/ipykernel_launcher.py -f [...]
```
Log files can be accessed via a special attribute.
```python { linenums="1" notest }
# [DataMember(data_store=/Users/gavin/repos/SciNexus/docs/data/demo-
# locked.sqlitedb, unique_id=logs/load_unaligned-progressive_align-write_db-
# pid8650.log)]
```
Each element in that list is a `DataMember` which you can use to get the data contents. The following
```python { notest }
print(dstore.logs[0].read()[:225])
```
Produces
```python { linenums="1" notest }
# 2019-07-24 14:42:56 Eratosthenes.local:8650 INFO system_details :
# system=Darwin Kernel Version 18.6.0: Thu Apr 25 23:16:27 PDT 2019;
# root:xnu-4903.261.4~2/RELEASE_X86_64 2019-07-24 14:42:56
# Eratosthenes.local:8650 INFO python
```
## Citations – giving credit to package developers
When apps declare citations, those citations are automatically saved alongside your results when you use `apply_to()`.
```python { linenums="1" notest }
import pathlib
import shutil
from citeable import Software
from scinexus import define_app, open_data_store
from cogent3 import get_app
from cogent3.app.typing import AlignedSeqsType
my_cite = Software(
author=["Doe, J"],
title="My Sequence Filter",
year=2025,
)
@define_app(cite=my_cite)
def strict_filter(val: AlignedSeqsType) -> AlignedSeqsType:
return val.omit_bad_seqs()
in_dstore = open_data_store("data/raw.zip", suffix="fa", limit=5)
out_dstore = open_data_store("cited_results", suffix="fa", mode="w")
loader = get_app("load_aligned", moltype="dna", format_name="fasta")
writer = get_app("write_seqs", data_store=out_dstore, format_name="fasta")
process = loader + strict_filter() + writer
result = process.apply_to(in_dstore)
result.write_bib("my_analysis.bib")
print(pathlib.Path("my_analysis.bib").read_text())
# @software{cogent3, author = {Huttley, Gavin and Caley, Katherine and
# Fotovat, Nabi and Ma, Stephen Ka-Wah and Koh, Moses and Morris, Richard and
# McArthur, Robert and McDonald, Daniel and Jaya, Fred and Maxwell, Peter and
# Martini, James and La, Thomas and Lang, Yapeng}, title = {{cogent3}: [...]
```
The `summary_citations` property returns a table of all citations stored in the data store (line 24). Export to BibTeX with `write_bib()` (line 26).
!!! note
`ReadOnlyDataStoreZipped` supports reading stored citations but not writing them.
cogent3-scinexus-b16bc06/docs/howto/use-with-coding-agents.md 0000664 0000000 0000000 00000007722 15257614025 0024236 0 ustar 00root root 0000000 0000000 # Use scinexus with a coding agent
!!! abstract ""
Install the `scinexus` agent skill, so that a coding assistant knows how the framework fits together before it writes any code for you. Works with any agent that reads skills, not one in particular.
## What the skill is
A directory of markdown: a `SKILL.md` carrying a worked loader, generic app and writer pipeline plus the rules that decide whether a first attempt works, and four files beside it that an agent opens only when a task reaches them -- the [llms.txt](../llms.txt) API summary, the file helpers, pipelines and data stores, and the hooks for a package built on `scinexus`.
It tells an assistant the things an API reference cannot: that a writer's `main` always receives `NotCompleted` values, that `mode="w"` creates a data store rather than emptying it, that `from __future__ import annotations` stops `define_app` working.
## Install it from this site
This site publishes the skill at the location the [agent skills standard](https://posit-dev.github.io/great-docs/user-guide/agent-skills.html) specifies, so a tool that follows that standard needs only the address this page is served from:
```bash
npx skills add https://scinexus.readthedocs.io/en/latest/
```
That reads the manifest at [`.well-known/agent-skills/index.json`](../.well-known/agent-skills/index.json) and installs what it lists. That link is relative, so following it from wherever you are reading this page gives the address to use, whatever prefix this site is served under. Where the files then land is the installing tool's business, and each assistant reads a different place:
| agent | directory |
| --- | --- |
| Claude Code | `.claude/skills/scinexus/` |
| Codex | `.codex/skills/scinexus/` |
| Cursor | `.cursor/skills/scinexus/` |
| GitHub Copilot | `.github/skills/scinexus/` |
| Windsurf | `.windsurf/skills/scinexus/` |
| OpenCode | `.opencode/skills/scinexus/` |
## Install it by hand
Nothing above is required. The files are plain markdown at a fixed address, so fetching them yourself works just as well -- substitute the directory your assistant reads:
```bash
mkdir -p .claude/skills/scinexus
base=https://scinexus.readthedocs.io/en/latest/.well-known/agent-skills/scinexus
for f in SKILL.md io.md pipelines.md extending.md llms.txt; do
curl -sSfo ".claude/skills/scinexus/$f" "$base/$f"
done
```
An agent that can fetch a URL itself needs even less: point it at `https://scinexus.readthedocs.io/en/latest/skill.md` and ask it to follow the file names inside.
## Install it as a Claude Code plugin
Claude Code can also install the skill from the repository, which keeps it up to date with the rest of the plugin machinery:
```
/plugin marketplace add cogent3/scinexus
/plugin install scinexus@scinexus
```
Worth knowing how that finds anything, since neither command names a skill. The first clones `github.com/cogent3/scinexus` and reads `.claude-plugin/marketplace.json` at its root, which lists one plugin named `scinexus` whose `source` is the repository root. The second reads `@`, so `scinexus@scinexus` is the plugin named `scinexus` from the marketplace named `scinexus`. Installing it copies that source, and Claude Code then finds skills where the manifest says they are, at `skills//SKILL.md`.
Because the source is the repository root, that copies the whole repository -- a couple of megabytes of package, tests and documentation for thirty kilobytes of markdown. The trade buys a single copy of the skill: the same files serve the plugin and this site, so the two cannot disagree. Either of the routes above fetches only the skill.
Remove it with:
```
/plugin uninstall scinexus@scinexus
/plugin marketplace remove scinexus
```
## Keeping it current
A skill is a copy, and copies go stale. The published files are rebuilt from the repository whenever these docs are, so re-running the install is what refreshes them. If your assistant starts asserting something this documentation contradicts, that is the first thing to check.
cogent3-scinexus-b16bc06/docs/howto/write-a-class-app.md 0000664 0000000 0000000 00000007225 15257614025 0023200 0 ustar 00root root 0000000 0000000 # Write a class app
!!! abstract ""
How to inherit from `scinexus` app base classes, or using the `define_app` decorator, specifying input/output type hints.
## Using inheritance from a base class
```python linenums="1"
from collections.abc import Callable
from citeable import Software
from scinexus import ComposableApp
my_cite = Software(
author=["Doe, J", "Smith, A"],
title="My Sequence Filter",
year=2025,
url="https://example.com/my-filter",
version="0.1.0",
)
class my_app( # (1)!
ComposableApp[str, str], # (2)!
cite=my_cite, # (3)!
):
def __init__(self, convert: Callable[[str], str]):
self.convert = convert
def main(self, val: str) -> str: # (4)!
return self.convert(val)
```
1. We suggest naming your apps using the PEP8 naming style for functions (lowercase separated by underscores) because the instances will be used like functions.
2. We type hint the input / output types with the base class.
3. We assign the citation in the class definition.
4. Your class **must** have a `main()` method with type hints specified for its first argument and its return type.
## Using the `define_app` decorator
How to use `@define_app` on a class with a `main()` method, configure it via `__init__` parameters, and control behaviour with the `app_type` parameter.
```python { linenums="1" notest }
from collections.abc import Callable
from citeable import Software
from scinexus import ComposableApp
my_cite = Software(
author=["Doe, J", "Smith, A"],
title="My Sequence Filter",
year=2025,
url="https://example.com/my-filter",
version="0.1.0",
)
@define_app(cite=my_cite) # (1)!
class my_app:
def __init__(self, convert: Callable[[str], str]):
self.convert = convert
def main(self, val: str) -> str: # (2)!
return self.convert(val)
```
1. The `define_app` decorator is used. You can specify the `app_type` here, which we don't in this case, and assign your citation.
2. Your class **must** have a `main()` method with type hints specified for its first argument and its return type.
### Specifying the app type
The `define_app` decorator has a default `app_type` of `"generic"`. This means the app does data transformation and does not load or write data. The supported app types are indicated by the `AppType` enum:
```python { linenums="1" notest }
from scinexus.composable import AppType
print(list(AppType))
# [, , , ]
```
If your app is not intended to be composed sequentially with other apps, set it to non-composable:
```python
from scinexus import define_app, AppType
@define_app(app_type=AppType.NON_COMPOSABLE)
class my_standalone_app:
def main(self, val: str) -> str:
return val.upper()
```
## Handling `NotCompleted` values
By default, apps skip `NotCompleted` inputs — they propagate through the pipeline without calling `main()`. If your app needs access to `NotCompleted` instances, set `skip_not_completed=False`:
```python
from scinexus import define_app, NotCompleted
@define_app(skip_not_completed=False)
class my_logger:
def main(self, val: str) -> str:
if isinstance(val, NotCompleted):
# handle the failure
...
return val
```
A writer does not need the flag — it is forced off, so a writer's `main()` always receives `NotCompleted` values and must handle them. See [Handle failures](handle-failures.md).
cogent3-scinexus-b16bc06/docs/howto/write-a-function-app.md 0000664 0000000 0000000 00000004271 15257614025 0023716 0 ustar 00root root 0000000 0000000 # Write a function app
!!! abstract ""
How to use `@define_app` on a plain function, specifying input/output type hints and turning function parameters into constructor arguments.
```python { linenums="1" notest }
from collections.abc import Callable
from citeable import Software
from scinexus import define_app, AppType
my_cite = Software(
author=["Doe, J", "Smith, A"],
title="My Sequence Filter",
year=2025,
url="https://example.com/my-filter",
version="0.1.0",
)
@define_app(
app_type=AppType.GENERIC, # (1)!
cite=my_cite, # (2)!
)
def my_app(val: str, convert: Callable[[str], str]) -> str: # (3)!
return convert(val)
app = my_app(str.upper)
print(app("hello world"))
# HELLO WORLD
```
1. We specify the `app_type` explicitly here.
2. We assign the citation in the class definition.
3. The function definition has type hints for its first argument and its return type.
!!! note
Your function can only have one required argument. It can have any number of optional arguments.
Pay attention to the order of arguments for the function! Every call to the app provides a new instance of `val`. Whereas `str.upper` is assigned to the variable `convert`. You can think of all of the other arguments as being arguments to a class constructor. Under the hood, `scinexus` caches these and injects them into each call of your function with new values of `val`.
cogent3-scinexus-b16bc06/docs/images/ 0000775 0000000 0000000 00000000000 15257614025 0017524 5 ustar 00root root 0000000 0000000 cogent3-scinexus-b16bc06/docs/images/logo-bw.png 0000664 0000000 0000000 00000002311 15257614025 0021575 0 ustar 00root root 0000000 0000000 ‰PNG
IHDR ä‚íÅ pHYs l l\ý. tEXtSoftware www.inkscape.org›î< VIDATH‰…Ö{ÈÞeðÏ÷éÝÞwSçœÊDg¥´EåaÐi£,g…‡y¬DHl”ž
+¡Lrh”a˜¥˜Ùô+ÃéŒrsYJ)uæ!sn+}·¬V©›]ýqß{x!ºàÇó;Þ×}]ßÃõ¤ª@’å8sð$.©ªÇû³%ø0ŽÄîZ¼�ç°wTÕ_’àLŒã |©ªžë]‚wâxlű¸9É�qþˆÛñ=lªªJ2
oÇRÜ–äIÃQx
ÁíIŽöÄ:¼oÂB,è ¯ÅûªêEÿ'’œƒOb föÌì•5À!x°ª¶÷ÖÀUµ
8;É“Œý�{'9ËðszgÆñOÜ……¼„7'YÔw0Ï$™Ó«ºGcM’³Fì“ä¬Ä.¸oÁ|LïIVc_LêdX‹°¸_7àüª¾3ŽËpï8bäù4üïïÉÆúñ3œ”ª’d_<�;°±¿|¾Ð�–c/œØ]Ý»qwU=Ú«\€âqlÁ)XWU˽°ßî žQUŸ¯!÷í—u`Wâ�‰ÁIö€ªzGô–¿Š�ã`½4ø4®ªª‡’üoÕ4"É\œ‡Sq^Á,�¾OõëËqnO¶#É;pfUmIòB’EÊí‚ñE;ãJ|½ƒú÷]¾¿ÓDyP'“$1YU[ú÷«ñÑAÇgó°MUõ~•ä‚îsqöÃ"Ü�ßàA|;ð|'Én½ý_Ùèoqè {ô�ŽÆà?Ò@?¼ãSýü:�43ú÷³{e�áÒ)ŸÄì�è´)‰f`o
ðqüÇëvkúâÁ†~ Ñ{þ”µ&h*Þ¤õ4®Å×4süj_`Í¢öÂûÑ,ìÀ¾Î©8.É⑵æaã ª¶bÎÐb’Œ×«êÞŽ×OðÙ^Õcø—FÝI
§í}3WÕ4)¬I´�Y÷œ–d®Á†$³ú³«p26k³Ysè74æ�‚«êÏPU/cS/|«†:Ú¤`¥ôÛ°6É7«êÖ$á&�Σ1ûãâá�$ó4̯L²ÛªêO4ü»i@žÞgjnü¡ÎüpÀóÚ€{vñ»5ÒFÆ-ÚšÜ�íá6ìªyßz\]UK“‰KñƒþÍ5Å¿UU[“ì§il…æûãem0nÃ>Uõt’h:ùG¯f¼ªÖ'™—dÂgªjCïÒ§’¯MâíšÖàQMØË4‡™Äecë¦ø”FÕY“žO²;þ�£«jÇ|ôḪ;Â7Úz¸_¿ZU÷%™�ém4œÜÜÖAEãrMŒ™šdJÌÕôµ´Ëä0�y‡hÞø‡Ø9ïÂõÚÌÿµæè'à—aÖkº¶nBûŸ±¤·üû8F#ÍmÎ3ñ #ƒoçã4M�ÓûËßª× É{5ñÍÓìgcOzO׎> ÏÒ´3�g±¢ª^ü/K3žæšâ1 IEND®B`‚ cogent3-scinexus-b16bc06/docs/images/logo-bw.svg 0000664 0000000 0000000 00000115123 15257614025 0021616 0 ustar 00root root 0000000 0000000
cogent3-scinexus-b16bc06/docs/images/logo-text-bw.svg 0000664 0000000 0000000 00000126214 15257614025 0022603 0 ustar 00root root 0000000 0000000
cogent3-scinexus-b16bc06/docs/images/logo-text-wb.svg 0000664 0000000 0000000 00000126301 15257614025 0022600 0 ustar 00root root 0000000 0000000
cogent3-scinexus-b16bc06/docs/images/logo-wb.png 0000664 0000000 0000000 00000002176 15257614025 0021606 0 ustar 00root root 0000000 0000000 ‰PNG
IHDR ä‚íÅ pHYs l l\ý. tEXtSoftware www.inkscape.org›î< IDATH‰…Öi¬�cðß9nÝÛ«¥›�¶–ŠV,¥–hKc%h«öH*"A[b‹6ˆ%„Zb-H,ICÔ´µ+mP»’ ZÔµ”¢(ãÃÌu�%Lòæœç¼ïûÌ3ó_æ4"Bű8ðÎÇuoöÆNX§~û
b.Âh`*ŽB;Þ¹ø°^:[à |�ýp'îÃ�XŒp3–!Лa